From a745df8a382be2e6d10dfe2816ea872b814c919b Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 03:07:11 -0700 Subject: [PATCH 01/59] Moved Core to the Direct Stream reads and appends. Core now reads and appends a Stream inside the Workflow Task. The bridge protos and the payload visitor follow its new commands and jobs, so a payload codec covers their bodies. --- temporalio/bridge/Cargo.lock | 1 + temporalio/bridge/_visitor.py | 49 +++- temporalio/bridge/proto/common/__init__.py | 2 + temporalio/bridge/proto/common/common_pb2.py | 8 +- temporalio/bridge/proto/common/common_pb2.pyi | 39 +++ .../bridge/proto/external_data/__init__.py | 13 +- .../proto/external_data/external_data_pb2.py | 28 +- .../proto/external_data/external_data_pb2.pyi | 248 ++++++++++++++++++ .../proto/workflow_activation/__init__.py | 8 + .../workflow_activation_pb2.py | 117 +++++---- .../workflow_activation_pb2.pyi | 177 +++++++++++++ .../proto/workflow_commands/__init__.py | 6 + .../workflow_commands_pb2.py | 143 +++++----- .../workflow_commands_pb2.pyi | 190 ++++++++++++++ temporalio/bridge/sdk-core | 2 +- 15 files changed, 895 insertions(+), 136 deletions(-) diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index 064e7b369..200959871 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -2927,6 +2927,7 @@ dependencies = [ "temporalio-client", "temporalio-common", "temporalio-macros", + "temporalio-streams", "thiserror", "tokio", "tokio-stream", diff --git a/temporalio/bridge/_visitor.py b/temporalio/bridge/_visitor.py index 8832b56bc..ada3052de 100644 --- a/temporalio/bridge/_visitor.py +++ b/temporalio/bridge/_visitor.py @@ -308,6 +308,32 @@ async def _visit_coresdk_workflow_activation_ResolveNexusOperation( if o.HasField("result"): await self._visit_coresdk_nexus_NexusOperationResult(fs, o.result) + async def _visit_temporal_api_stream_v1_StreamRecord( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("body"): + await self._visit_temporal_api_common_v1_Payload(fs, o.body) + + async def _visit_coresdk_workflow_activation_StreamReadEnd( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("failure"): + await self._visit_temporal_api_failure_v1_Failure(fs, o.failure) + + async def _visit_coresdk_workflow_activation_ResolveStreamRead( + self, fs: VisitorFunctions, o: Any + ): + for v in o.records: + await self._visit_temporal_api_stream_v1_StreamRecord(fs, v) + if o.HasField("end"): + await self._visit_coresdk_workflow_activation_StreamReadEnd(fs, o.end) + + async def _visit_coresdk_workflow_activation_ResolveStreamAppend( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("failed"): + await self._visit_temporal_api_failure_v1_Failure(fs, o.failed) + async def _visit_coresdk_workflow_activation_WorkflowActivationJob( self, fs: VisitorFunctions, o: Any ): @@ -353,6 +379,14 @@ async def _visit_coresdk_workflow_activation_WorkflowActivationJob( await self._visit_coresdk_workflow_activation_ResolveNexusOperation( fs, o.resolve_nexus_operation ) + elif o.HasField("resolve_stream_read"): + await self._visit_coresdk_workflow_activation_ResolveStreamRead( + fs, o.resolve_stream_read + ) + elif o.HasField("resolve_stream_append"): + await self._visit_coresdk_workflow_activation_ResolveStreamAppend( + fs, o.resolve_stream_append + ) async def _visit_coresdk_workflow_activation_WorkflowActivation( self, fs: VisitorFunctions, o: Any @@ -492,6 +526,11 @@ async def _visit_coresdk_workflow_commands_ScheduleNexusOperation( if o.HasField("input"): await self._visit_nexus_operation_input_payload(fs, o.input) + async def _visit_coresdk_workflow_commands_AppendToStream( + self, fs: VisitorFunctions, o: Any + ): + await self._visit_payload_container(fs, o.bodies) + async def _visit_coresdk_workflow_commands_WorkflowCommand( self, fs: VisitorFunctions, o: Any ): @@ -551,6 +590,10 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand( await self._visit_coresdk_workflow_commands_ScheduleNexusOperation( fs, o.schedule_nexus_operation ) + elif o.HasField("append_to_stream"): + await self._visit_coresdk_workflow_commands_AppendToStream( + fs, o.append_to_stream + ) async def _visit_coresdk_workflow_completion_Success( self, fs: VisitorFunctions, o: Any @@ -681,12 +724,6 @@ async def _visit_temporal_api_streamservice_v1_ReadRecordsRequest( ) -> None: pass - async def _visit_temporal_api_stream_v1_StreamRecord( - self, fs: VisitorFunctions, o: Any - ): - if o.HasField("body"): - await self._visit_temporal_api_common_v1_Payload(fs, o.body) - async def _visit_temporal_api_streamservice_v1_ReadRecordsResponse( self, fs: VisitorFunctions, o: Any ): diff --git a/temporalio/bridge/proto/common/__init__.py b/temporalio/bridge/proto/common/__init__.py index a8506090d..b9b0cce02 100644 --- a/temporalio/bridge/proto/common/__init__.py +++ b/temporalio/bridge/proto/common/__init__.py @@ -1,6 +1,7 @@ from .common_pb2 import ( ExternalStorageMetrics, NamespacedWorkflowExecution, + StreamReadStop, VersioningIntent, WorkerDeploymentVersion, ) @@ -8,6 +9,7 @@ __all__ = [ "ExternalStorageMetrics", "NamespacedWorkflowExecution", + "StreamReadStop", "VersioningIntent", "WorkerDeploymentVersion", ] diff --git a/temporalio/bridge/proto/common/common_pb2.py b/temporalio/bridge/proto/common/common_pb2.py index 0b9a5c816..e492e93c1 100644 --- a/temporalio/bridge/proto/common/common_pb2.py +++ b/temporalio/bridge/proto/common/common_pb2.py @@ -16,7 +16,7 @@ from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n%temporal/sdk/core/common/common.proto\x12\x0e\x63oresdk.common\x1a\x1egoogle/protobuf/duration.proto"U\n\x1bNamespacedWorkflowExecution\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"D\n\x17WorkerDeploymentVersion\x12\x17\n\x0f\x64\x65ployment_name\x18\x01 \x01(\t\x12\x10\n\x08\x62uild_id\x18\x02 \x01(\t"\x92\x01\n\x16\x45xternalStorageMetrics\x12\x15\n\rpayload_count\x18\x01 \x01(\x04\x12\x18\n\x10total_size_bytes\x18\x02 \x01(\x04\x12\x31\n\x0etotal_duration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x14\n\x0c\x64river_names\x18\x04 \x03(\t*@\n\x10VersioningIntent\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCOMPATIBLE\x10\x01\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x02\x42,\xea\x02)Temporalio::Internal::Bridge::Api::Commonb\x06proto3' + b'\n%temporal/sdk/core/common/common.proto\x12\x0e\x63oresdk.common\x1a\x1egoogle/protobuf/duration.proto"U\n\x1bNamespacedWorkflowExecution\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"D\n\x17WorkerDeploymentVersion\x12\x17\n\x0f\x64\x65ployment_name\x18\x01 \x01(\t\x12\x10\n\x08\x62uild_id\x18\x02 \x01(\t"\x92\x01\n\x16\x45xternalStorageMetrics\x12\x15\n\rpayload_count\x18\x01 \x01(\x04\x12\x18\n\x10total_size_bytes\x18\x02 \x01(\x04\x12\x31\n\x0etotal_duration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x14\n\x0c\x64river_names\x18\x04 \x03(\t*\xc8\x01\n\x0eStreamReadStop\x12 \n\x1cSTREAM_READ_STOP_UNSPECIFIED\x10\x00\x12\x19\n\x15STREAM_READ_STOP_DONE\x10\x01\x12 \n\x1cSTREAM_READ_STOP_MAX_RECORDS\x10\x02\x12\x1e\n\x1aSTREAM_READ_STOP_MAX_BYTES\x10\x03\x12\x19\n\x15STREAM_READ_STOP_IDLE\x10\x04\x12\x1c\n\x18STREAM_READ_STOP_TIMEOUT\x10\x05*@\n\x10VersioningIntent\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCOMPATIBLE\x10\x01\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x02\x42,\xea\x02)Temporalio::Internal::Bridge::Api::Commonb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -26,8 +26,10 @@ DESCRIPTOR._serialized_options = ( b"\352\002)Temporalio::Internal::Bridge::Api::Common" ) - _VERSIONINGINTENT._serialized_start = 395 - _VERSIONINGINTENT._serialized_end = 459 + _STREAMREADSTOP._serialized_start = 396 + _STREAMREADSTOP._serialized_end = 596 + _VERSIONINGINTENT._serialized_start = 598 + _VERSIONINGINTENT._serialized_end = 662 _NAMESPACEDWORKFLOWEXECUTION._serialized_start = 89 _NAMESPACEDWORKFLOWEXECUTION._serialized_end = 174 _WORKERDEPLOYMENTVERSION._serialized_start = 176 diff --git a/temporalio/bridge/proto/common/common_pb2.pyi b/temporalio/bridge/proto/common/common_pb2.pyi index 7c2b05702..da85e098d 100644 --- a/temporalio/bridge/proto/common/common_pb2.pyi +++ b/temporalio/bridge/proto/common/common_pb2.pyi @@ -21,6 +21,45 @@ else: DESCRIPTOR: google.protobuf.descriptor.FileDescriptor +class _StreamReadStop: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _StreamReadStopEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + _StreamReadStop.ValueType + ], + builtins.type, +): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + STREAM_READ_STOP_UNSPECIFIED: _StreamReadStop.ValueType # 0 + STREAM_READ_STOP_DONE: _StreamReadStop.ValueType # 1 + """The Stream is closed, and the read gave its last Record.""" + STREAM_READ_STOP_MAX_RECORDS: _StreamReadStop.ValueType # 2 + """The read gave `max_records` Records.""" + STREAM_READ_STOP_MAX_BYTES: _StreamReadStop.ValueType # 3 + """The next Record would pass `max_bytes`.""" + STREAM_READ_STOP_IDLE: _StreamReadStop.ValueType # 4 + """No Record came for `idle_timeout`.""" + STREAM_READ_STOP_TIMEOUT: _StreamReadStop.ValueType # 5 + """`timeout` passed since the read started.""" + +class StreamReadStop(_StreamReadStop, metaclass=_StreamReadStopEnumTypeWrapper): + """Why a read of a Stream ended. A read that failed has no stop.""" + +STREAM_READ_STOP_UNSPECIFIED: StreamReadStop.ValueType # 0 +STREAM_READ_STOP_DONE: StreamReadStop.ValueType # 1 +"""The Stream is closed, and the read gave its last Record.""" +STREAM_READ_STOP_MAX_RECORDS: StreamReadStop.ValueType # 2 +"""The read gave `max_records` Records.""" +STREAM_READ_STOP_MAX_BYTES: StreamReadStop.ValueType # 3 +"""The next Record would pass `max_bytes`.""" +STREAM_READ_STOP_IDLE: StreamReadStop.ValueType # 4 +"""No Record came for `idle_timeout`.""" +STREAM_READ_STOP_TIMEOUT: StreamReadStop.ValueType # 5 +"""`timeout` passed since the read started.""" +global___StreamReadStop = StreamReadStop + class _VersioningIntent: ValueType = typing.NewType("ValueType", builtins.int) V: typing_extensions.TypeAlias = ValueType diff --git a/temporalio/bridge/proto/external_data/__init__.py b/temporalio/bridge/proto/external_data/__init__.py index 836e97ef4..2230b1cbb 100644 --- a/temporalio/bridge/proto/external_data/__init__.py +++ b/temporalio/bridge/proto/external_data/__init__.py @@ -1,6 +1,17 @@ -from .external_data_pb2 import LocalActivityMarkerData, PatchedMarkerData +from .external_data_pb2 import ( + LocalActivityMarkerData, + PatchedMarkerData, + StreamAppendEntry, + StreamMarkerData, + StreamReadEnded, + StreamReadEntry, +) __all__ = [ "LocalActivityMarkerData", "PatchedMarkerData", + "StreamAppendEntry", + "StreamMarkerData", + "StreamReadEnded", + "StreamReadEntry", ] diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.py b/temporalio/bridge/proto/external_data/external_data_pb2.py index 4bda6f514..188b86456 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.py +++ b/temporalio/bridge/proto/external_data/external_data_pb2.py @@ -16,8 +16,18 @@ from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from temporalio.api.failure.v1 import ( + message_pb2 as temporal_dot_api_dot_failure_dot_v1_dot_message__pb2, +) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) +from temporalio.bridge.proto.common import ( + common_pb2 as temporal_dot_sdk_dot_core_dot_common_dot_common__pb2, +) + DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\xb2\x02\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x10\x61\x63tivation_index\x18\x08 \x01(\x04H\x00\x88\x01\x01\x42\x13\n\x11_activation_index"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08\x42\x32\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' + b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a%temporal/sdk/core/common/common.proto"\xb2\x02\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x10\x61\x63tivation_index\x18\x08 \x01(\x04H\x00\x88\x01\x01\x42\x13\n\x11_activation_index"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\x84\x01\n\x10StreamMarkerData\x12\x35\n\x05reads\x18\x01 \x03(\x0b\x32&.coresdk.external_data.StreamReadEntry\x12\x39\n\x07\x61ppends\x18\x02 \x03(\x0b\x32(.coresdk.external_data.StreamAppendEntry"\xcb\x02\n\x0fStreamReadEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12\x14\n\x0c\x61\x66ter_offset\x18\x04 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x05 \x01(\x0c\x12\r\n\x05\x63ount\x18\x06 \x01(\r\x12\x0e\n\x06\x64igest\x18\x07 \x01(\x0c\x12\x35\n\x07records\x18\x08 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x13\n\x0bpage_counts\x18\t \x03(\r\x12\x18\n\x10page_activations\x18\n \x03(\r\x12\x33\n\x03\x65nd\x18\x0b \x01(\x0b\x32&.coresdk.external_data.StreamReadEnded"\xa9\x01\n\x0fStreamReadEnded\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x05 \x01(\r"\xc5\x01\n\x11StreamAppendEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x14\n\x0c\x66irst_offset\x18\x03 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x04 \x01(\x0c\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\rB2\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -27,8 +37,16 @@ DESCRIPTOR._serialized_options = ( b"\352\002/Temporalio::Internal::Bridge::Api::ExternalData" ) - _LOCALACTIVITYMARKERDATA._serialized_start = 144 - _LOCALACTIVITYMARKERDATA._serialized_end = 450 - _PATCHEDMARKERDATA._serialized_start = 452 - _PATCHEDMARKERDATA._serialized_end = 503 + _LOCALACTIVITYMARKERDATA._serialized_start = 260 + _LOCALACTIVITYMARKERDATA._serialized_end = 566 + _PATCHEDMARKERDATA._serialized_start = 568 + _PATCHEDMARKERDATA._serialized_end = 619 + _STREAMMARKERDATA._serialized_start = 622 + _STREAMMARKERDATA._serialized_end = 754 + _STREAMREADENTRY._serialized_start = 757 + _STREAMREADENTRY._serialized_end = 1088 + _STREAMREADENDED._serialized_start = 1091 + _STREAMREADENDED._serialized_end = 1260 + _STREAMAPPENDENTRY._serialized_start = 1263 + _STREAMAPPENDENTRY._serialized_end = 1460 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.pyi b/temporalio/bridge/proto/external_data/external_data_pb2.pyi index edcd4581a..d2e5f4788 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.pyi +++ b/temporalio/bridge/proto/external_data/external_data_pb2.pyi @@ -4,13 +4,19 @@ isort:skip_file """ import builtins +import collections.abc import typing import google.protobuf.descriptor import google.protobuf.duration_pb2 +import google.protobuf.internal.containers import google.protobuf.message import google.protobuf.timestamp_pb2 +import temporalio.api.failure.v1.message_pb2 +import temporalio.api.stream.v1.message_pb2 +import temporalio.bridge.proto.common.common_pb2 + DESCRIPTOR: google.protobuf.descriptor.FileDescriptor @typing.final @@ -139,3 +145,245 @@ class PatchedMarkerData(google.protobuf.message.Message): ) -> None: ... global___PatchedMarkerData = PatchedMarkerData + +@typing.final +class StreamMarkerData(google.protobuf.message.Message): + """The data of the marker `core_stream`, which Core records once for each workflow task in which + the workflow got a Stream result. It is in the detail `data` as `binary/protobuf`, so that a + marker with Records stays small. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + READS_FIELD_NUMBER: builtins.int + APPENDS_FIELD_NUMBER: builtins.int + @property + def reads( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamReadEntry + ]: ... + @property + def appends( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamAppendEntry + ]: ... + def __init__( + self, + *, + reads: collections.abc.Iterable[global___StreamReadEntry] | None = ..., + appends: collections.abc.Iterable[global___StreamAppendEntry] | None = ..., + ) -> None: ... + def ClearField( + self, field_name: typing.Literal["appends", b"appends", "reads", b"reads"] + ) -> None: ... + +global___StreamMarkerData = StreamMarkerData + +@typing.final +class StreamReadEntry(google.protobuf.message.Message): + """What one read gave the workflow in one workflow task.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + AFTER_OFFSET_FIELD_NUMBER: builtins.int + LAST_OFFSET_FIELD_NUMBER: builtins.int + COUNT_FIELD_NUMBER: builtins.int + DIGEST_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + PAGE_COUNTS_FIELD_NUMBER: builtins.int + PAGE_ACTIVATIONS_FIELD_NUMBER: builtins.int + END_FIELD_NUMBER: builtins.int + seq: builtins.int + from_offset: builtins.bytes + """The `from_offset` of the read, so a replay can check that the read is the same.""" + after_offset: builtins.bytes + """The offset before the first Record of this task. A replay reads after it.""" + last_offset: builtins.bytes + """The offset of the last Record of this task. Empty when the task got no Record.""" + count: builtins.int + """The number of Records of this task.""" + digest: builtins.bytes + """The digest of the Records of this task. Empty when `records` holds them.""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: + """The Records of this task, only for a read with its Records in history.""" + + @property + def page_counts( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: + """The number of Records of each page, in order. A page ends where its count ends.""" + + @property + def page_activations( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: + """The activation of each page in the task. The first is the index, and each later one is the + gap to the one before. + """ + + @property + def end(self) -> global___StreamReadEnded: + """Set when the read ended in this task.""" + + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + from_offset: builtins.bytes = ..., + after_offset: builtins.bytes = ..., + last_offset: builtins.bytes = ..., + count: builtins.int = ..., + digest: builtins.bytes = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + page_counts: collections.abc.Iterable[builtins.int] | None = ..., + page_activations: collections.abc.Iterable[builtins.int] | None = ..., + end: global___StreamReadEnded | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["end", b"end", "stream", b"stream"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "after_offset", + b"after_offset", + "count", + b"count", + "digest", + b"digest", + "end", + b"end", + "from_offset", + b"from_offset", + "last_offset", + b"last_offset", + "page_activations", + b"page_activations", + "page_counts", + b"page_counts", + "records", + b"records", + "seq", + b"seq", + "stream", + b"stream", + ], + ) -> None: ... + +global___StreamReadEntry = StreamReadEntry + +@typing.final +class StreamReadEnded(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LAST_OFFSET_FIELD_NUMBER: builtins.int + STOP_FIELD_NUMBER: builtins.int + DONE_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + ACTIVATION_FIELD_NUMBER: builtins.int + last_offset: builtins.bytes + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType + done: builtins.bool + activation: builtins.int + """The activation in the task that got the end.""" + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + last_offset: builtins.bytes = ..., + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType = ..., + done: builtins.bool = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + activation: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failure", b"failure"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "activation", + b"activation", + "done", + b"done", + "failure", + b"failure", + "last_offset", + b"last_offset", + "stop", + b"stop", + ], + ) -> None: ... + +global___StreamReadEnded = StreamReadEnded + +@typing.final +class StreamAppendEntry(google.protobuf.message.Message): + """The result of one append, in the workflow task whose activation got it.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + FIRST_OFFSET_FIELD_NUMBER: builtins.int + LAST_OFFSET_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + ACTIVATION_FIELD_NUMBER: builtins.int + seq: builtins.int + first_offset: builtins.bytes + last_offset: builtins.bytes + activation: builtins.int + """The activation in the task that got the result.""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + first_offset: builtins.bytes = ..., + last_offset: builtins.bytes = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + activation: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failure", b"failure", "stream", b"stream"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "activation", + b"activation", + "failure", + b"failure", + "first_offset", + b"first_offset", + "last_offset", + b"last_offset", + "seq", + b"seq", + "stream", + b"stream", + ], + ) -> None: ... + +global___StreamAppendEntry = StreamAppendEntry diff --git a/temporalio/bridge/proto/workflow_activation/__init__.py b/temporalio/bridge/proto/workflow_activation/__init__.py index 5c178e218..fb214394a 100644 --- a/temporalio/bridge/proto/workflow_activation/__init__.py +++ b/temporalio/bridge/proto/workflow_activation/__init__.py @@ -16,7 +16,11 @@ ResolveNexusOperationStart, ResolveRequestCancelExternalWorkflow, ResolveSignalExternalWorkflow, + ResolveStreamAppend, + ResolveStreamRead, SignalWorkflow, + StreamAppended, + StreamReadEnd, UpdateRandomSeed, WorkflowActivation, WorkflowActivationJob, @@ -40,7 +44,11 @@ "ResolveNexusOperationStart", "ResolveRequestCancelExternalWorkflow", "ResolveSignalExternalWorkflow", + "ResolveStreamAppend", + "ResolveStreamRead", "SignalWorkflow", + "StreamAppended", + "StreamReadEnd", "UpdateRandomSeed", "WorkflowActivation", "WorkflowActivationJob", diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py index 589119c72..cac6b3961 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py @@ -32,6 +32,9 @@ from temporalio.api.nexus.v1 import ( message_pb2 as temporal_dot_api_dot_nexus_dot_v1_dot_message__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.api.update.v1 import ( message_pb2 as temporal_dot_api_dot_update_dot_v1_dot_message__pb2, ) @@ -49,7 +52,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\xe0\n\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\xe3\x0b\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x12\x65\n&propagated_nexus_serialization_context\x18\x1b \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"\xb1\x01\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause"\xb8\x01\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause"\xb2\x03\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x12\x65\n&propagated_nexus_serialization_context\x18\x08 \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\nB8\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' + b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\x82\x0c\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12M\n\x13resolve_stream_read\x18\x11 \x01(\x0b\x32..coresdk.workflow_activation.ResolveStreamReadH\x00\x12Q\n\x15resolve_stream_append\x18\x12 \x01(\x0b\x32\x30.coresdk.workflow_activation.ResolveStreamAppendH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\xe3\x0b\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x12\x65\n&propagated_nexus_serialization_context\x18\x1b \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"\xb1\x01\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause"\xb8\x01\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause"\xb2\x03\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x12\x65\n&propagated_nexus_serialization_context\x18\x08 \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\n"\x90\x01\n\x11ResolveStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x35\n\x07records\x18\x02 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x37\n\x03\x65nd\x18\x03 \x01(\x0b\x32*.coresdk.workflow_activation.StreamReadEnd"\x93\x01\n\rStreamReadEnd\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\xa1\x01\n\x13ResolveStreamAppend\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12?\n\x08\x61ppended\x18\x02 \x01(\x0b\x32+.coresdk.workflow_activation.StreamAppendedH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status";\n\x0eStreamAppended\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x02 \x01(\x0c\x42\x38\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -67,56 +70,64 @@ _SIGNALWORKFLOW_HEADERSENTRY._serialized_options = b"8\001" _DOUPDATE_HEADERSENTRY._options = None _DOUPDATE_HEADERSENTRY._serialized_options = b"8\001" - _WORKFLOWACTIVATION._serialized_start = 611 - _WORKFLOWACTIVATION._serialized_end = 1157 - _WORKFLOWACTIVATIONJOB._serialized_start = 1160 - _WORKFLOWACTIVATIONJOB._serialized_end = 2536 - _INITIALIZEWORKFLOW._serialized_start = 2539 - _INITIALIZEWORKFLOW._serialized_end = 4046 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 3967 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 4046 - _FIRETIMER._serialized_start = 4048 - _FIRETIMER._serialized_end = 4072 - _RESOLVEACTIVITY._serialized_start = 4074 - _RESOLVEACTIVITY._serialized_end = 4183 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4186 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4523 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4525 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4584 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4587 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 4753 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 4755 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 4851 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 4853 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 4958 - _UPDATERANDOMSEED._serialized_start = 4960 - _UPDATERANDOMSEED._serialized_end = 5003 - _QUERYWORKFLOW._serialized_start = 5006 - _QUERYWORKFLOW._serialized_end = 5266 - _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 3967 - _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 4046 - _CANCELWORKFLOW._serialized_start = 5268 - _CANCELWORKFLOW._serialized_end = 5300 - _SIGNALWORKFLOW._serialized_start = 5303 - _SIGNALWORKFLOW._serialized_end = 5592 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 3967 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 4046 - _NOTIFYHASPATCH._serialized_start = 5594 - _NOTIFYHASPATCH._serialized_end = 5628 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5631 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 5808 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 5811 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 5995 - _DOUPDATE._serialized_start = 5998 - _DOUPDATE._serialized_end = 6432 - _DOUPDATE_HEADERSENTRY._serialized_start = 3967 - _DOUPDATE_HEADERSENTRY._serialized_end = 4046 - _RESOLVENEXUSOPERATIONSTART._serialized_start = 6435 - _RESOLVENEXUSOPERATIONSTART._serialized_end = 6589 - _RESOLVENEXUSOPERATION._serialized_start = 6591 - _RESOLVENEXUSOPERATION._serialized_end = 6680 - _REMOVEFROMCACHE._serialized_start = 6683 - _REMOVEFROMCACHE._serialized_end = 7035 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 6797 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 7035 + _WORKFLOWACTIVATION._serialized_start = 649 + _WORKFLOWACTIVATION._serialized_end = 1195 + _WORKFLOWACTIVATIONJOB._serialized_start = 1198 + _WORKFLOWACTIVATIONJOB._serialized_end = 2736 + _INITIALIZEWORKFLOW._serialized_start = 2739 + _INITIALIZEWORKFLOW._serialized_end = 4246 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 4167 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 4246 + _FIRETIMER._serialized_start = 4248 + _FIRETIMER._serialized_end = 4272 + _RESOLVEACTIVITY._serialized_start = 4274 + _RESOLVEACTIVITY._serialized_end = 4383 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4386 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4723 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4725 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4784 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4787 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 4953 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 4955 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 5051 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 5053 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 5158 + _UPDATERANDOMSEED._serialized_start = 5160 + _UPDATERANDOMSEED._serialized_end = 5203 + _QUERYWORKFLOW._serialized_start = 5206 + _QUERYWORKFLOW._serialized_end = 5466 + _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 4167 + _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 4246 + _CANCELWORKFLOW._serialized_start = 5468 + _CANCELWORKFLOW._serialized_end = 5500 + _SIGNALWORKFLOW._serialized_start = 5503 + _SIGNALWORKFLOW._serialized_end = 5792 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 4167 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 4246 + _NOTIFYHASPATCH._serialized_start = 5794 + _NOTIFYHASPATCH._serialized_end = 5828 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5831 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 6008 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 6011 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 6195 + _DOUPDATE._serialized_start = 6198 + _DOUPDATE._serialized_end = 6632 + _DOUPDATE_HEADERSENTRY._serialized_start = 4167 + _DOUPDATE_HEADERSENTRY._serialized_end = 4246 + _RESOLVENEXUSOPERATIONSTART._serialized_start = 6635 + _RESOLVENEXUSOPERATIONSTART._serialized_end = 6789 + _RESOLVENEXUSOPERATION._serialized_start = 6791 + _RESOLVENEXUSOPERATION._serialized_end = 6880 + _REMOVEFROMCACHE._serialized_start = 6883 + _REMOVEFROMCACHE._serialized_end = 7235 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 6997 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 7235 + _RESOLVESTREAMREAD._serialized_start = 7238 + _RESOLVESTREAMREAD._serialized_end = 7382 + _STREAMREADEND._serialized_start = 7385 + _STREAMREADEND._serialized_end = 7532 + _RESOLVESTREAMAPPEND._serialized_start = 7535 + _RESOLVESTREAMAPPEND._serialized_end = 7696 + _STREAMAPPENDED._serialized_start = 7698 + _STREAMAPPENDED._serialized_end = 7757 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi index af1a2e38a..8886676e0 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi @@ -23,6 +23,7 @@ import temporalio.api.enums.v1.failed_cause_pb2 import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 import temporalio.api.nexus.v1.message_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.update.v1.message_pb2 import temporalio.bridge.proto.activity_result.activity_result_pb2 import temporalio.bridge.proto.child_workflow.child_workflow_pb2 @@ -244,6 +245,8 @@ class WorkflowActivationJob(google.protobuf.message.Message): DO_UPDATE_FIELD_NUMBER: builtins.int RESOLVE_NEXUS_OPERATION_START_FIELD_NUMBER: builtins.int RESOLVE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int + RESOLVE_STREAM_READ_FIELD_NUMBER: builtins.int + RESOLVE_STREAM_APPEND_FIELD_NUMBER: builtins.int REMOVE_FROM_CACHE_FIELD_NUMBER: builtins.int @property def initialize_workflow(self) -> global___InitializeWorkflow: @@ -318,6 +321,14 @@ class WorkflowActivationJob(google.protobuf.message.Message): def resolve_nexus_operation(self) -> global___ResolveNexusOperation: """A nexus operation resolved.""" + @property + def resolve_stream_read(self) -> global___ResolveStreamRead: + """A read of a Stream gave Records, ended, or both.""" + + @property + def resolve_stream_append(self) -> global___ResolveStreamAppend: + """An append to a Stream resolved.""" + @property def remove_from_cache(self) -> global___RemoveFromCache: """Remove the workflow identified by the [WorkflowActivation] containing this job from the @@ -347,6 +358,8 @@ class WorkflowActivationJob(google.protobuf.message.Message): do_update: global___DoUpdate | None = ..., resolve_nexus_operation_start: global___ResolveNexusOperationStart | None = ..., resolve_nexus_operation: global___ResolveNexusOperation | None = ..., + resolve_stream_read: global___ResolveStreamRead | None = ..., + resolve_stream_append: global___ResolveStreamAppend | None = ..., remove_from_cache: global___RemoveFromCache | None = ..., ) -> None: ... def HasField( @@ -380,6 +393,10 @@ class WorkflowActivationJob(google.protobuf.message.Message): b"resolve_request_cancel_external_workflow", "resolve_signal_external_workflow", b"resolve_signal_external_workflow", + "resolve_stream_append", + b"resolve_stream_append", + "resolve_stream_read", + b"resolve_stream_read", "signal_workflow", b"signal_workflow", "update_random_seed", @@ -419,6 +436,10 @@ class WorkflowActivationJob(google.protobuf.message.Message): b"resolve_request_cancel_external_workflow", "resolve_signal_external_workflow", b"resolve_signal_external_workflow", + "resolve_stream_append", + b"resolve_stream_append", + "resolve_stream_read", + b"resolve_stream_read", "signal_workflow", b"signal_workflow", "update_random_seed", @@ -446,6 +467,8 @@ class WorkflowActivationJob(google.protobuf.message.Message): "do_update", "resolve_nexus_operation_start", "resolve_nexus_operation", + "resolve_stream_read", + "resolve_stream_append", "remove_from_cache", ] | None @@ -1639,3 +1662,157 @@ class RemoveFromCache(google.protobuf.message.Message): ) -> None: ... global___RemoveFromCache = RemoveFromCache + +@typing.final +class ResolveStreamRead(google.protobuf.message.Message): + """The Records that a read of a Stream gave since its last job, and its end if it ended.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + END_FIELD_NUMBER: builtins.int + seq: builtins.int + """The sequence number of the `StartStreamRead`""" + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: ... + @property + def end(self) -> global___StreamReadEnd: + """Set on the last job of a read.""" + + def __init__( + self, + *, + seq: builtins.int = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + end: global___StreamReadEnd | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["end", b"end"]) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal["end", b"end", "records", b"records", "seq", b"seq"], + ) -> None: ... + +global___ResolveStreamRead = ResolveStreamRead + +@typing.final +class StreamReadEnd(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LAST_OFFSET_FIELD_NUMBER: builtins.int + STOP_FIELD_NUMBER: builtins.int + DONE_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + last_offset: builtins.bytes + """The offset of the last Record that the workflow got, or `from_offset` if it got none.""" + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType + """Why the read ended. Unspecified when it failed.""" + done: builtins.bool + """True if the Stream is closed and the read gave its last Record.""" + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: + """Set if the read failed.""" + + def __init__( + self, + *, + last_offset: builtins.bytes = ..., + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType = ..., + done: builtins.bool = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failure", b"failure"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "done", + b"done", + "failure", + b"failure", + "last_offset", + b"last_offset", + "stop", + b"stop", + ], + ) -> None: ... + +global___StreamReadEnd = StreamReadEnd + +@typing.final +class ResolveStreamAppend(google.protobuf.message.Message): + """The result of an `AppendToStream`.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + APPENDED_FIELD_NUMBER: builtins.int + FAILED_FIELD_NUMBER: builtins.int + seq: builtins.int + """The sequence number of the `AppendToStream`""" + @property + def appended(self) -> global___StreamAppended: ... + @property + def failed(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + seq: builtins.int = ..., + appended: global___StreamAppended | None = ..., + failed: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing.Literal[ + "appended", b"appended", "failed", b"failed", "status", b"status" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "appended", + b"appended", + "failed", + b"failed", + "seq", + b"seq", + "status", + b"status", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing.Literal["status", b"status"] + ) -> typing.Literal["appended", "failed"] | None: ... + +global___ResolveStreamAppend = ResolveStreamAppend + +@typing.final +class StreamAppended(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FIRST_OFFSET_FIELD_NUMBER: builtins.int + LAST_OFFSET_FIELD_NUMBER: builtins.int + first_offset: builtins.bytes + last_offset: builtins.bytes + def __init__( + self, + *, + first_offset: builtins.bytes = ..., + last_offset: builtins.bytes = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing.Literal[ + "first_offset", b"first_offset", "last_offset", b"last_offset" + ], + ) -> None: ... + +global___StreamAppended = StreamAppended diff --git a/temporalio/bridge/proto/workflow_commands/__init__.py b/temporalio/bridge/proto/workflow_commands/__init__.py index f599917af..92fcecf8b 100644 --- a/temporalio/bridge/proto/workflow_commands/__init__.py +++ b/temporalio/bridge/proto/workflow_commands/__init__.py @@ -1,9 +1,11 @@ from .workflow_commands_pb2 import ( ActivityCancellationType, + AppendToStream, CancelChildWorkflowExecution, CancelSignalWorkflow, CancelTimer, CancelWorkflowExecution, + CloseStreamRead, CompleteWorkflowExecution, ContinueAsNewWorkflowExecution, FailWorkflowExecution, @@ -20,6 +22,7 @@ SetPatchMarker, SignalExternalWorkflowExecution, StartChildWorkflowExecution, + StartStreamRead, StartTimer, UpdateResponse, UpsertWorkflowSearchAttributes, @@ -28,10 +31,12 @@ __all__ = [ "ActivityCancellationType", + "AppendToStream", "CancelChildWorkflowExecution", "CancelSignalWorkflow", "CancelTimer", "CancelWorkflowExecution", + "CloseStreamRead", "CompleteWorkflowExecution", "ContinueAsNewWorkflowExecution", "FailWorkflowExecution", @@ -48,6 +53,7 @@ "SetPatchMarker", "SignalExternalWorkflowExecution", "StartChildWorkflowExecution", + "StartStreamRead", "StartTimer", "UpdateResponse", "UpsertWorkflowSearchAttributes", diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py index 3573f36b1..f06e2de58 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py @@ -32,6 +32,9 @@ from temporalio.api.sdk.v1 import ( user_metadata_pb2 as temporal_dot_api_dot_sdk_dot_v1_dot_user__metadata__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.api.workflow.v1 import ( message_pb2 as temporal_dot_api_dot_workflow_dot_v1_dot_message__pb2, ) @@ -46,7 +49,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\xa9\x10\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x42\t\n\x07variant"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x93\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12#\n\x1binclude_arguments_in_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xe1\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12I\n\x13versioning_override\x18\x15 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' + b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a$temporal/api/stream/v1/message.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\x82\x12\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x12G\n\x11start_stream_read\x18\x17 \x01(\x0b\x32*.coresdk.workflow_commands.StartStreamReadH\x00\x12G\n\x11\x63lose_stream_read\x18\x18 \x01(\x0b\x32*.coresdk.workflow_commands.CloseStreamReadH\x00\x12\x45\n\x10\x61ppend_to_stream\x18\x19 \x01(\x0b\x32).coresdk.workflow_commands.AppendToStreamH\x00\x42\t\n\x07variant"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x93\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12#\n\x1binclude_arguments_in_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xe1\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12I\n\x13versioning_override\x18\x15 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x87\x02\n\x0fStartStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12*\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12/\n\x0cidle_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bmax_records\x18\x06 \x01(\r\x12\x11\n\tmax_bytes\x18\x07 \x01(\x04\x12\x1a\n\x12records_in_history\x18\x08 \x01(\x08"\x1e\n\x0f\x43loseStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xc7\x01\n\x0e\x41ppendToStream\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12/\n\x06\x62odies\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x32\n\x08producer\x18\x04 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12\x10\n\x08sequence\x18\x05 \x01(\x03*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -72,70 +75,76 @@ _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_options = b"8\001" _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._options = None _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_options = b"8\001" - _ACTIVITYCANCELLATIONTYPE._serialized_start = 8785 - _ACTIVITYCANCELLATIONTYPE._serialized_end = 8873 - _WORKFLOWCOMMAND._serialized_start = 558 - _WORKFLOWCOMMAND._serialized_end = 2647 - _STARTTIMER._serialized_start = 2649 - _STARTTIMER._serialized_end = 2732 - _CANCELTIMER._serialized_start = 2734 - _CANCELTIMER._serialized_end = 2760 - _SCHEDULEACTIVITY._serialized_start = 2763 - _SCHEDULEACTIVITY._serialized_end = 3587 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3508 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3587 - _SCHEDULELOCALACTIVITY._serialized_start = 3590 - _SCHEDULELOCALACTIVITY._serialized_end = 4377 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3508 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3587 - _REQUESTCANCELACTIVITY._serialized_start = 4379 - _REQUESTCANCELACTIVITY._serialized_end = 4415 - _REQUESTCANCELLOCALACTIVITY._serialized_start = 4417 - _REQUESTCANCELLOCALACTIVITY._serialized_end = 4458 - _QUERYRESULT._serialized_start = 4461 - _QUERYRESULT._serialized_end = 4617 - _QUERYSUCCESS._serialized_start = 4619 - _QUERYSUCCESS._serialized_end = 4684 - _COMPLETEWORKFLOWEXECUTION._serialized_start = 4686 - _COMPLETEWORKFLOWEXECUTION._serialized_end = 4762 - _FAILWORKFLOWEXECUTION._serialized_start = 4764 - _FAILWORKFLOWEXECUTION._serialized_end = 4838 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 4841 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 5755 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5598 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5674 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3508 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3587 - _CANCELWORKFLOWEXECUTION._serialized_start = 5757 - _CANCELWORKFLOWEXECUTION._serialized_end = 5833 - _SETPATCHMARKER._serialized_start = 5835 - _SETPATCHMARKER._serialized_end = 5889 - _STARTCHILDWORKFLOWEXECUTION._serialized_start = 5892 - _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7141 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3508 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3587 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5598 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5674 - _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7143 - _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7217 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7220 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7362 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7365 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 7764 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3508 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3587 - _CANCELSIGNALWORKFLOW._serialized_start = 7766 - _CANCELSIGNALWORKFLOW._serialized_end = 7801 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 7803 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 7904 - _MODIFYWORKFLOWPROPERTIES._serialized_start = 7906 - _MODIFYWORKFLOWPROPERTIES._serialized_end = 7985 - _UPDATERESPONSE._serialized_start = 7988 - _UPDATERESPONSE._serialized_end = 8198 - _SCHEDULENEXUSOPERATION._serialized_start = 8201 - _SCHEDULENEXUSOPERATION._serialized_end = 8739 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 8689 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 8739 - _REQUESTCANCELNEXUSOPERATION._serialized_start = 8741 - _REQUESTCANCELNEXUSOPERATION._serialized_end = 8783 + _ACTIVITYCANCELLATIONTYPE._serialized_start = 9540 + _ACTIVITYCANCELLATIONTYPE._serialized_end = 9628 + _WORKFLOWCOMMAND._serialized_start = 596 + _WORKFLOWCOMMAND._serialized_end = 2902 + _STARTTIMER._serialized_start = 2904 + _STARTTIMER._serialized_end = 2987 + _CANCELTIMER._serialized_start = 2989 + _CANCELTIMER._serialized_end = 3015 + _SCHEDULEACTIVITY._serialized_start = 3018 + _SCHEDULEACTIVITY._serialized_end = 3842 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3763 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3842 + _SCHEDULELOCALACTIVITY._serialized_start = 3845 + _SCHEDULELOCALACTIVITY._serialized_end = 4632 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3763 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3842 + _REQUESTCANCELACTIVITY._serialized_start = 4634 + _REQUESTCANCELACTIVITY._serialized_end = 4670 + _REQUESTCANCELLOCALACTIVITY._serialized_start = 4672 + _REQUESTCANCELLOCALACTIVITY._serialized_end = 4713 + _QUERYRESULT._serialized_start = 4716 + _QUERYRESULT._serialized_end = 4872 + _QUERYSUCCESS._serialized_start = 4874 + _QUERYSUCCESS._serialized_end = 4939 + _COMPLETEWORKFLOWEXECUTION._serialized_start = 4941 + _COMPLETEWORKFLOWEXECUTION._serialized_end = 5017 + _FAILWORKFLOWEXECUTION._serialized_start = 5019 + _FAILWORKFLOWEXECUTION._serialized_end = 5093 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 5096 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 6010 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5853 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5929 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3763 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3842 + _CANCELWORKFLOWEXECUTION._serialized_start = 6012 + _CANCELWORKFLOWEXECUTION._serialized_end = 6088 + _SETPATCHMARKER._serialized_start = 6090 + _SETPATCHMARKER._serialized_end = 6144 + _STARTCHILDWORKFLOWEXECUTION._serialized_start = 6147 + _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7396 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3763 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3842 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5853 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5929 + _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7398 + _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7472 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7475 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7617 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7620 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 8019 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3763 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3842 + _CANCELSIGNALWORKFLOW._serialized_start = 8021 + _CANCELSIGNALWORKFLOW._serialized_end = 8056 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 8058 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 8159 + _MODIFYWORKFLOWPROPERTIES._serialized_start = 8161 + _MODIFYWORKFLOWPROPERTIES._serialized_end = 8240 + _UPDATERESPONSE._serialized_start = 8243 + _UPDATERESPONSE._serialized_end = 8453 + _SCHEDULENEXUSOPERATION._serialized_start = 8456 + _SCHEDULENEXUSOPERATION._serialized_end = 8994 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 8944 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 8994 + _REQUESTCANCELNEXUSOPERATION._serialized_start = 8996 + _REQUESTCANCELNEXUSOPERATION._serialized_end = 9038 + _STARTSTREAMREAD._serialized_start = 9041 + _STARTSTREAMREAD._serialized_end = 9304 + _CLOSESTREAMREAD._serialized_start = 9306 + _CLOSESTREAMREAD._serialized_end = 9336 + _APPENDTOSTREAM._serialized_start = 9339 + _APPENDTOSTREAM._serialized_end = 9538 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi index 81c47c0b0..4c9e7bb63 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi @@ -25,6 +25,7 @@ import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 import temporalio.api.sdk.v1.event_group_marker_pb2 import temporalio.api.sdk.v1.user_metadata_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.workflow.v1.message_pb2 import temporalio.bridge.proto.child_workflow.child_workflow_pb2 import temporalio.bridge.proto.common.common_pb2 @@ -105,6 +106,9 @@ class WorkflowCommand(google.protobuf.message.Message): UPDATE_RESPONSE_FIELD_NUMBER: builtins.int SCHEDULE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int REQUEST_CANCEL_NEXUS_OPERATION_FIELD_NUMBER: builtins.int + START_STREAM_READ_FIELD_NUMBER: builtins.int + CLOSE_STREAM_READ_FIELD_NUMBER: builtins.int + APPEND_TO_STREAM_FIELD_NUMBER: builtins.int @property def user_metadata(self) -> temporalio.api.sdk.v1.user_metadata_pb2.UserMetadata: """User metadata that may or may not be persisted into history depending on the command type. @@ -181,6 +185,12 @@ class WorkflowCommand(google.protobuf.message.Message): def request_cancel_nexus_operation( self, ) -> global___RequestCancelNexusOperation: ... + @property + def start_stream_read(self) -> global___StartStreamRead: ... + @property + def close_stream_read(self) -> global___CloseStreamRead: ... + @property + def append_to_stream(self) -> global___AppendToStream: ... def __init__( self, *, @@ -219,10 +229,15 @@ class WorkflowCommand(google.protobuf.message.Message): schedule_nexus_operation: global___ScheduleNexusOperation | None = ..., request_cancel_nexus_operation: global___RequestCancelNexusOperation | None = ..., + start_stream_read: global___StartStreamRead | None = ..., + close_stream_read: global___CloseStreamRead | None = ..., + append_to_stream: global___AppendToStream | None = ..., ) -> None: ... def HasField( self, field_name: typing.Literal[ + "append_to_stream", + b"append_to_stream", "cancel_child_workflow_execution", b"cancel_child_workflow_execution", "cancel_signal_workflow", @@ -231,6 +246,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"cancel_timer", "cancel_workflow_execution", b"cancel_workflow_execution", + "close_stream_read", + b"close_stream_read", "complete_workflow_execution", b"complete_workflow_execution", "continue_as_new_workflow_execution", @@ -261,6 +278,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"signal_external_workflow_execution", "start_child_workflow_execution", b"start_child_workflow_execution", + "start_stream_read", + b"start_stream_read", "start_timer", b"start_timer", "update_response", @@ -276,6 +295,8 @@ class WorkflowCommand(google.protobuf.message.Message): def ClearField( self, field_name: typing.Literal[ + "append_to_stream", + b"append_to_stream", "cancel_child_workflow_execution", b"cancel_child_workflow_execution", "cancel_signal_workflow", @@ -284,6 +305,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"cancel_timer", "cancel_workflow_execution", b"cancel_workflow_execution", + "close_stream_read", + b"close_stream_read", "complete_workflow_execution", b"complete_workflow_execution", "continue_as_new_workflow_execution", @@ -316,6 +339,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"signal_external_workflow_execution", "start_child_workflow_execution", b"start_child_workflow_execution", + "start_stream_read", + b"start_stream_read", "start_timer", b"start_timer", "update_response", @@ -354,6 +379,9 @@ class WorkflowCommand(google.protobuf.message.Message): "update_response", "schedule_nexus_operation", "request_cancel_nexus_operation", + "start_stream_read", + "close_stream_read", + "append_to_stream", ] | None ): ... @@ -1931,3 +1959,165 @@ class RequestCancelNexusOperation(google.protobuf.message.Message): def ClearField(self, field_name: typing.Literal["seq", b"seq"]) -> None: ... global___RequestCancelNexusOperation = RequestCancelNexusOperation + +@typing.final +class StartStreamRead(google.protobuf.message.Message): + """Start a read of a Stream. Core reads the Store itself and gives each page of Records to the + workflow in a `ResolveStreamRead` job, until the read ends or the workflow closes it. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + TIMEOUT_FIELD_NUMBER: builtins.int + IDLE_TIMEOUT_FIELD_NUMBER: builtins.int + MAX_RECORDS_FIELD_NUMBER: builtins.int + MAX_BYTES_FIELD_NUMBER: builtins.int + RECORDS_IN_HISTORY_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's incremental sequence number of Stream reads, used as the read's identifier""" + from_offset: builtins.bytes + """The read starts after this offset. Empty means the start of the Stream.""" + max_records: builtins.int + """The most Records the read gives. Zero means no cap.""" + max_bytes: builtins.int + """The most bytes of Records the read gives. The first Record is always given. Zero means no + cap. + """ + records_in_history: builtins.bool + """If set, the Records go into the workflow history, so a replay reads no Store.""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def timeout(self) -> google.protobuf.duration_pb2.Duration: + """The longest time the read may take. Unset means no bound.""" + + @property + def idle_timeout(self) -> google.protobuf.duration_pb2.Duration: + """The read ends when no Record comes for this long. Unset means one second.""" + + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + from_offset: builtins.bytes = ..., + timeout: google.protobuf.duration_pb2.Duration | None = ..., + idle_timeout: google.protobuf.duration_pb2.Duration | None = ..., + max_records: builtins.int = ..., + max_bytes: builtins.int = ..., + records_in_history: builtins.bool = ..., + ) -> None: ... + def HasField( + self, + field_name: typing.Literal[ + "idle_timeout", b"idle_timeout", "stream", b"stream", "timeout", b"timeout" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "from_offset", + b"from_offset", + "idle_timeout", + b"idle_timeout", + "max_bytes", + b"max_bytes", + "max_records", + b"max_records", + "records_in_history", + b"records_in_history", + "seq", + b"seq", + "stream", + b"stream", + "timeout", + b"timeout", + ], + ) -> None: ... + +global___StartStreamRead = StartStreamRead + +@typing.final +class CloseStreamRead(google.protobuf.message.Message): + """Stop a read that `StartStreamRead` started. The workflow gets no job for it.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's sequence number as passed to `StartStreamRead`""" + def __init__( + self, + *, + seq: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["seq", b"seq"]) -> None: ... + +global___CloseStreamRead = CloseStreamRead + +@typing.final +class AppendToStream(google.protobuf.message.Message): + """Append Records to a Stream. Core calls the Store itself and gives the result in a + `ResolveStreamAppend` job. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + BODIES_FIELD_NUMBER: builtins.int + PRODUCER_FIELD_NUMBER: builtins.int + SEQUENCE_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's incremental sequence number of Stream appends, used as the append's identifier""" + sequence: builtins.int + """The `sequence` of the Store request, as the caller gives it. Zero means Core uses `seq`.""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def bodies( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.common.v1.message_pb2.Payload + ]: + """The bodies of the Records, in order""" + + @property + def producer(self) -> temporalio.api.stream.v1.message_pb2.Producer: + """The workflow's Producer, of kind workflow, with its id and run id and no epoch. Core sets the + epoch to the current workflow task. + """ + + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + bodies: collections.abc.Iterable[temporalio.api.common.v1.message_pb2.Payload] + | None = ..., + producer: temporalio.api.stream.v1.message_pb2.Producer | None = ..., + sequence: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["producer", b"producer", "stream", b"stream"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "bodies", + b"bodies", + "producer", + b"producer", + "seq", + b"seq", + "sequence", + b"sequence", + "stream", + b"stream", + ], + ) -> None: ... + +global___AppendToStream = AppendToStream diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 9b121cf32..d92cd4704 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 9b121cf32dcd7920a067a04fb61bbb000178290b +Subproject commit d92cd470496551cf38f927dedc4deb803e5e0731 From 436ad1ad3b5bda99be2042ab416665937a9923a7 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 03:08:53 -0700 Subject: [PATCH 02/59] Moved Core to the Stream WIT with the Workflow `iter` read. The WIT declares `iter` as a helper method of the Workflow Stream. Only that file differs from the pin before, so the bridge needs no rebuild. --- temporalio/bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index d92cd4704..92081b7cc 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit d92cd470496551cf38f927dedc4deb803e5e0731 +Subproject commit 92081b7ccb1596f2f6f9247094a49b94832fcff5 From d3788ded81f5369a3dd5debaaf348319c7b8178c Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 03:08:53 -0700 Subject: [PATCH 03/59] Pinned the Stream generator to nexgen with sync helpers and regenerated. The Stream WIT marks `iter` with `@nexus.helper sync=true` and returns a support class. Only a fork of nexgen knows these tags until its `main` has them. --- .github/workflows/ci.yml | 4 ++-- scripts/gen_nexus_system_api.py | 8 +++++--- .../workflow/_resources/stream.py | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c60bb3316..91342ce5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -228,8 +228,8 @@ jobs: - run: uv tool install poethepoet - run: uv sync --all-extras - run: cargo install --locked nexgen --version 0.2.8 --features advanced --force - # The Stream package needs the `main` of nexgen, which no release has yet. - - run: cargo install --locked --git https://github.com/temporalio/nexgen --rev 9653a0f5a845df0c5059554f1b5813db6a74cd1a --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen + # The Stream package needs `@nexus.helper` with `sync=true`, which only a fork of nexgen has yet. + - run: cargo install --locked --git https://github.com/moetemp/nexgen --rev 42c7e43799f97c837346f1562db13493d9869858 --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen - run: echo "NEX_GEN_STREAM_BIN=$RUNNER_TEMP/nexgen-stream/bin/nexgen" >> "$GITHUB_ENV" - run: poe build-develop - run: poe gen-protos diff --git a/scripts/gen_nexus_system_api.py b/scripts/gen_nexus_system_api.py index 48998fc7c..065805135 100644 --- a/scripts/gen_nexus_system_api.py +++ b/scripts/gen_nexus_system_api.py @@ -28,8 +28,9 @@ workflow_init_path = base_dir / "temporalio" / "workflow" / "__init__.py" NEX_GEN_VERSION = "0.2.8" # The Stream package needs a newer nexgen, which rejects the other WITs. One pin is enough -# again once the api WITs and a nexgen release agree. No release has nexgen commit 9653a0f5, -# the `main` of nexgen, yet, so NEX_GEN_STREAM_BIN points at a build of it. +# again once the api WITs and a nexgen release agree. The Stream WIT uses `@nexus.helper` with +# `sync=true` and `@nexus.support-type`, which only the fork https://github.com/moetemp/nexgen +# has, at commit 42c7e437. So NEX_GEN_STREAM_BIN points at a build of that commit. NEX_GEN_STREAM_VERSION = "0.2.12" proto_files = [ gen_protos.api_proto_dir @@ -87,7 +88,8 @@ def nex_gen_stream_command() -> list[str]: bin_path = os.environ.get("NEX_GEN_STREAM_BIN") if not bin_path: raise RuntimeError( - "NEX_GEN_STREAM_BIN must point at a nexgen build of commit 9653a0f5" + "NEX_GEN_STREAM_BIN must point at a nexgen build of commit 42c7e437 " + "of https://github.com/moetemp/nexgen" ) version = subprocess.check_output([bin_path, "--version"], text=True).strip() if version != f"nexgen {NEX_GEN_STREAM_VERSION}": diff --git a/temporalio/nexus/system/stream_service/workflow/_resources/stream.py b/temporalio/nexus/system/stream_service/workflow/_resources/stream.py index df548d252..36018dbd2 100644 --- a/temporalio/nexus/system/stream_service/workflow/_resources/stream.py +++ b/temporalio/nexus/system/stream_service/workflow/_resources/stream.py @@ -2,6 +2,7 @@ from __future__ import annotations +import datetime import types import typing @@ -96,6 +97,24 @@ async def describe( handle = await _describe_stream(request) return await handle + def iter( + self, + timeout: datetime.timedelta | None = None, + from_offset: bytes | None = None, + idle_timeout: datetime.timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, + ) -> _support.StreamReading[BodyT]: + """Read each Record as it arrives, up to a stop.""" + return _support.iter_stream( + self, + timeout=timeout, + from_offset=from_offset, + idle_timeout=idle_timeout, + max_records=max_records, + max_bytes=max_bytes, + ) + @typing.overload def get_stream( From f16ae1426570fa8c96a0009604b7bda23d13bdc8 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 03:11:20 -0700 Subject: [PATCH 04/59] Gave Core the Stream runtime of each Store on the bridge. Core reads and appends a Stream of Workflow code through the runtime of its Store. The Worker and the Replayer pass one for each Store of the client's Stream config that has one. --- temporalio/bridge/src/streams.rs | 2 +- temporalio/bridge/src/worker.rs | 14 ++++++++++++++ temporalio/bridge/worker.py | 3 +++ temporalio/client/_stream_store.py | 31 ++++++++++++++++++++++++++---- temporalio/worker/_replayer.py | 5 ++++- temporalio/worker/_stream_store.py | 7 +++++++ temporalio/worker/_worker.py | 3 ++- 7 files changed, 58 insertions(+), 7 deletions(-) diff --git a/temporalio/bridge/src/streams.rs b/temporalio/bridge/src/streams.rs index 5eb56cc72..e005a5afc 100644 --- a/temporalio/bridge/src/streams.rs +++ b/temporalio/bridge/src/streams.rs @@ -16,7 +16,7 @@ pyo3::create_exception!(temporal_sdk_bridge, StreamCallError, PyException); #[cfg(feature = "streams")] #[pyclass(frozen)] pub struct StreamRuntimeRef { - streams: std::sync::Arc, + pub(crate) streams: std::sync::Arc, runtime: runtime::Runtime, } diff --git a/temporalio/bridge/src/worker.rs b/temporalio/bridge/src/worker.rs index 39cdc5e77..ad7f7c611 100644 --- a/temporalio/bridge/src/worker.rs +++ b/temporalio/bridge/src/worker.rs @@ -65,6 +65,10 @@ pub struct WorkerConfig { plugins: Vec<(String, String)>, storage_drivers: HashSet, disable_payload_error_limit: bool, + /// The Stream runtime of each Store that has one, by Store name. Core reads and appends a + /// Stream of Workflow code through them. + #[cfg(feature = "streams")] + stream_runtimes: Vec<(String, Py)>, } #[derive(FromPyObject)] @@ -718,6 +722,15 @@ fn convert_worker_config( ) -> PyResult { let converted_tuner = convert_tuner_holder(conf.tuner, task_locals)?; let converted_versioning_strategy = convert_versioning_strategy(conf.versioning_strategy); + // A build with no Stream runtime has no Store that Core can call, so Core gets none. + #[cfg(feature = "streams")] + let stream_runtimes = temporalio_sdk_core::StreamRuntimes::new( + conf.stream_runtimes + .into_iter() + .map(|(store, runtime)| (store, runtime.get().streams.clone())), + ); + #[cfg(not(feature = "streams"))] + let stream_runtimes = temporalio_sdk_core::StreamRuntimes::default(); temporalio_sdk_core::WorkerConfig::builder() .namespace(conf.namespace) .task_queue(conf.task_queue) @@ -777,6 +790,7 @@ fn convert_worker_config( .collect::>(), ) .disable_payload_error_limit(conf.disable_payload_error_limit) + .stream_runtimes(stream_runtimes) .build() .map_err(|err| PyValueError::new_err(format!("Invalid worker config: {err}"))) } diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index fa5f337ca..647c09878 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -8,6 +8,7 @@ from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass from typing import ( + Any, TypeAlias, ) @@ -60,6 +61,8 @@ class WorkerConfig: plugins: Sequence[tuple[str, str]] storage_drivers: set[str] disable_payload_error_limit: bool + stream_runtimes: Sequence[tuple[str, Any]] = () + """The bridge's object of each Store's Stream runtime, by Store name.""" @dataclass diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index c2f1364c9..be550f34c 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -237,6 +237,18 @@ async def describe_stream( """See base class.""" return await self._call_bridge("DescribeStream", request) + def _get_stream_runtime(self) -> temporalio.bridge.streams.StreamRuntime: + """Give the Stream runtime of the bridge, and build it at the first use. + + Raises: + temporalio.exceptions.StreamConfigError: This build of temporalio has no + Stream runtime, or not this Store. + """ + if self._stream_runtime is None: + runtime = self._runtime or temporalio.runtime.Runtime.default() + self._stream_runtime = self._new_stream_runtime(runtime._core_runtime) + return self._stream_runtime + async def _call_bridge( self, operation: str, request: google.protobuf.message.Message ) -> Any: @@ -247,11 +259,8 @@ async def _call_bridge( temporalio.exceptions.StreamConfigError: This build of temporalio has no Stream runtime, or not this Store. """ - if self._stream_runtime is None: - runtime = self._runtime or temporalio.runtime.Runtime.default() - self._stream_runtime = self._new_stream_runtime(runtime._core_runtime) # The bridge takes and gives the messages in their wire form. - response = await self._stream_runtime.call( + response = await self._get_stream_runtime().call( operation, request.SerializeToString() ) return _STREAM_OPERATIONS[operation].response_type.FromString(response) @@ -532,6 +541,20 @@ def _get_store_by_name(self, name: str) -> StreamStore: ) return store + def _bridge_stream_runtimes( + self, + ) -> list[tuple[str, temporalio.bridge.streams.StreamRuntime]]: + """Give the Stream runtime of each Store that runs on the bridge, by Store name. + + Core reads and appends a Stream of Workflow code through these runtimes. A Store + that a user wrote in Python has none. + """ + return [ + (store.name, store._get_stream_runtime()) + for store in self.stores + if isinstance(store, _BridgeStreamStore) + ] + def _store_for( self, ref: temporalio.nexus.system.stream_service.StreamRef[Any], diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index 7ad6bdeaa..e179facc7 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -22,7 +22,7 @@ from ..common import HeaderCodecBehavior from ._interceptor import Interceptor -from ._stream_store import _WorkflowStreamStore +from ._stream_store import _bridge_stream_runtimes, _WorkflowStreamStore from ._worker import load_default_build_id from ._workflow import _WorkflowWorker from ._workflow_instance import ( @@ -368,6 +368,9 @@ def on_eviction_hook( (plugin.name(), plugin.version()) for plugin in self.plugins ], storage_drivers=storage_driver_types, + stream_runtimes=_bridge_stream_runtimes( + self._config.get("stream_config") + ), ), ) bridge_worker_scope = bridge_worker diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index daaac73e6..0365861be 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -41,6 +41,13 @@ def _no_stream_config_error() -> temporalio.exceptions.ApplicationError: # pyri _ORDERED_OPERATIONS = frozenset({"AppendRecords", "CloseStream"}) +def _bridge_stream_runtimes(config: StreamConfig | None) -> list[tuple[str, Any]]: + """Give Core the bridge's object of each Store's Stream runtime, by Store name.""" + if config is None: + return [] + return [(name, runtime._ref) for name, runtime in config._bridge_stream_runtimes()] + + def _workflow_target() -> StorageDriverWorkflowInfo: # External storage builds the same value for a Workflow. info = temporalio.workflow.info() diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index bbf68e2bf..7c870853e 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -34,7 +34,7 @@ from ._interceptor import Interceptor from ._nexus import _NexusWorker from ._plugin import Plugin -from ._stream_store import _WorkflowStreamStore +from ._stream_store import _bridge_stream_runtimes, _WorkflowStreamStore from ._tuning import WorkerTuner from ._workflow import ( _DEFAULT_WORKFLOW_TASK_EXTERNAL_STORAGE_CONCURRENCY, @@ -724,6 +724,7 @@ def check_activity(activity: str): disable_payload_error_limit=config.get( "disable_payload_error_limit", False ), + stream_runtimes=_bridge_stream_runtimes(stream_config), ), ) From 3ae1427f4c2f557d380e2387d3aa3121b4cc6e5d Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 03:18:08 -0700 Subject: [PATCH 05/59] Added the Workflow `iter` read, which Core runs in the Workflow Task. Core reads the Store and gives each page to the read in an activation, so a read costs one marker for each Workflow Task. `StreamConfig.workflow_read_replay_from` decides whether History holds the Records. --- temporalio/bridge/worker.py | 1 + temporalio/client/__init__.py | 2 + temporalio/client/_stream_store.py | 49 ++++ temporalio/common.py | 13 + temporalio/nexus/system/_support.py | 261 ++++++++++++++++++++ temporalio/worker/_command_aware_visitor.py | 27 ++ temporalio/worker/_stream_store.py | 82 +++++- temporalio/worker/_workflow_instance.py | 92 ++++++- temporalio/workflow/__init__.py | 5 + temporalio/workflow/_context.py | 6 + 10 files changed, 536 insertions(+), 2 deletions(-) diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index 647c09878..616e75690 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -371,6 +371,7 @@ async def encode_completion( skip_search_attributes=True, skip_headers=not encode_headers, concurrency_limit=storage_concurrency_limit, + skip_stream_bodies=True, ).visit( _Visitor(data_converter._external_store_payload_sequence), completion, diff --git a/temporalio/client/__init__.py b/temporalio/client/__init__.py index 6fa6dd710..8f567ea21 100644 --- a/temporalio/client/__init__.py +++ b/temporalio/client/__init__.py @@ -187,6 +187,7 @@ ) from ._stream_store import ( StreamConfig, + StreamReplaySource, StreamSelectContext, StreamStore, StreamStoreCallOptions, @@ -357,6 +358,7 @@ "CloudOperationsClient", "Plugin", "StreamConfig", + "StreamReplaySource", "StreamSelectContext", "StreamStore", "StreamStoreCallOptions", diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index be550f34c..76913a435 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -1,6 +1,7 @@ from __future__ import annotations import dataclasses +import enum from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable, Sequence from datetime import timedelta @@ -25,6 +26,39 @@ import temporalio.bridge.streams +class StreamReplaySource(enum.Enum): + """Where a replay of a read of Workflow code gets the Records of the read. + + :py:attr:`StreamConfig.workflow_read_replay_from` holds it. It decides what a new read + records in History. A replay follows what History holds, so a change of the setting + doesn't break a Workflow that is running. Client and Activity code have no History, so + the setting doesn't touch them. + + .. warning:: + This API is experimental and subject to change. + """ + + HISTORY = "history" + """The marker in History holds the Records of the read with their bodies. + + A replay takes the Records from History, so it reads no Store and needs no Stream + config. Each Record adds to History, so a read with no ``max_bytes`` gets the cap + :py:data:`temporalio.common.DEFAULT_STREAM_MAX_BYTES`. + """ + + STORE = "store" + """The marker in History holds no Record and no body. This is the default. + + For each Workflow Task of a read, the marker holds the range of offsets, the count + and a digest of the Records that the Workflow got. A replay reads the Records from the + Store again and checks the count and the digest. So the replay needs the Stream config + and the Records. The Records must stay in the Store, and Retention must keep them, for + as long as the Workflow can replay. When they can't be read, the Workflow Task fails + with a message that starts with ``StreamReplayError``. A replay on a Worker with no + cache reads the Store again on every Workflow Task. + """ + + @dataclasses.dataclass(frozen=True) class StreamStoreCallOptions: """Options of the Local Activity that calls a Stream Store for Workflow code. @@ -489,6 +523,21 @@ class StreamConfig: Local Activity with their timeouts and retry policy. """ + workflow_read_replay_from: StreamReplaySource = dataclasses.field( + default=StreamReplaySource.STORE + ) + """Where a replay of a read of Workflow code gets the Records of the read. + + It applies to every read of Workflow code on a Worker whose client has this config, + and decides what a new read records in History. :py:attr:`StreamReplaySource.STORE`, + the default, puts only the range of offsets, the count and a digest of the Records + into History, so a replay reads them from the Store again. The Records must then stay + in the Store for as long as the Workflow can replay. The option + :py:attr:`StreamReplaySource.HISTORY` puts the Records with their bodies into + History, so a replay needs no Store. A replay follows the marker in History and not + this setting, so a Workflow that is running replays after a change of it. + """ + _store_map: dict[str, StreamStore] = dataclasses.field( init=False, repr=False, compare=False ) diff --git a/temporalio/common.py b/temporalio/common.py index 8bcc66a4a..af1af74e4 100644 --- a/temporalio/common.py +++ b/temporalio/common.py @@ -1455,6 +1455,19 @@ class HeaderCodecBehavior(IntEnum): """Only automatically encode and decode headers in workflow activation encoding and decoding.""" +DEFAULT_STREAM_MAX_BYTES = 256 * 1024 +"""The most bytes of the Records of one read of a Stream in Workflow code, when History +holds the Records. + +It applies when the read's ``max_bytes`` is ``None``. By default a read of Workflow code has +no such cap, since no body goes into History. It has this cap when +``StreamConfig.workflow_read_replay_from`` is the option ``HISTORY``. + +.. warning:: + This API is experimental and subject to change. +""" + + @dataclass class _CompositeEvent: # pyright: ignore[reportUnusedClass] # This should always be present, but is sometimes lazily set internally diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index 55f53b59a..6f96358ff 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -1,5 +1,8 @@ +import asyncio +import collections import collections.abc import dataclasses +import enum import functools import typing import urllib.parse @@ -8,6 +11,7 @@ import google.protobuf.duration_pb2 import google.protobuf.timestamp_pb2 +import typing_extensions import temporalio.activity import temporalio.api.common.v1.message_pb2 as common_pb2 @@ -20,6 +24,7 @@ import temporalio.nexus.system if typing.TYPE_CHECKING: + import temporalio.api.stream.v1 from temporalio.nexus.system.stream_service.models import ( Producer, ReadRecordsRequest, @@ -170,6 +175,262 @@ def resume_read_request( return dataclasses.replace(request, from_offset=item.offset) +class StreamReadStop(enum.IntEnum): + """Why a read of Workflow code ended. + + .. warning:: + This API is experimental and subject to change. + """ + + Done = 1 + """The Stream is closed, and the read gave its last Record.""" + + MaxRecords = 2 + """The read gave ``max_records`` Records.""" + + MaxBytes = 3 + """The next Record would pass ``max_bytes``.""" + + Idle = 4 + """No Record came for ``idle_timeout``.""" + + Timeout = 5 + """``timeout`` passed since the read started.""" + + +@dataclasses.dataclass(frozen=True) +class _StreamReadOptions: + """The checked arguments of one read, as `iter_stream` takes them.""" + + from_offset: bytes | None + timeout: timedelta | None + idle_timeout: timedelta | None + max_records: int | None + max_bytes: int | None + + +class StreamReading(typing.Generic[_BodyT]): + """One read of a Stream in Workflow code. It gives each Record as it arrives. + + Get it from ``iter`` of a Workflow Stream, and read it with ``async for``. The read + starts at the first step of the loop and ends at a stop. Then :py:attr:`stop` and + :py:attr:`done` tell why, and :py:attr:`last_offset` is the ``from_offset`` of the + next read. See ``iter`` for the stops and the costs of a read. + + .. warning:: + This API is experimental and subject to change. + """ + + last_offset: bytes | None + """The offset of the last Record that the loop got, or ``from_offset`` when it got + none. A new read from it loses and repeats no Record.""" + + stop: StreamReadStop | None + """Why the read ended. ``None`` while it runs, and after a failure or a close.""" + + done: bool + """Whether the Stream is closed and the read gave its last Record.""" + + def __init__(self, stream: typing.Any, options: _StreamReadOptions) -> None: + """Keep the call. Use ``iter`` of a Workflow Stream instead.""" + self._stream = stream + self._options = options + self.last_offset = options.from_offset + self.stop = None + self.done = False + self._seq: int | None = None + self._decode: ( + collections.abc.Callable[ + [temporalio.api.stream.v1.StreamRecord], StreamRecord[_BodyT] + ] + | None + ) = None + self._records: collections.deque[temporalio.api.stream.v1.StreamRecord] = ( + collections.deque() + ) + self._end: tuple[StreamReadStop | None, bool, BaseException | None] | None = ( + None + ) + self._waiter: asyncio.Future[None] | None = None + self._finished = False + + def __aiter__(self) -> typing_extensions.Self: + """Give this read, since it is its own iterator.""" + return self + + async def __anext__(self) -> "StreamRecord[_BodyT]": + """Give the next Record, and start the read at the first call. + + Raises: + StopAsyncIteration: The read ended at a stop, or the code closed it. + temporalio.exceptions.StreamError: The read failed. + """ + import temporalio.workflow + + if self._finished: + raise StopAsyncIteration + if self._seq is None: + temporalio.workflow._Runtime.current().workflow_start_stream_read(self) + while not self._records and self._end is None: + self._waiter = asyncio.get_running_loop().create_future() + try: + await self._waiter + finally: + self._waiter = None + if self._finished: + raise StopAsyncIteration + if self._records: + record = self._records.popleft() + if self._decode is None: + raise RuntimeError("The Stream read has no decoder") + self.last_offset = record.offset + return self._decode(record) + assert self._end is not None + stop, done, error = self._end + self._finished = True + if error is not None: + raise error + self.stop = stop + self.done = done + raise StopAsyncIteration + + async def aclose(self) -> None: + """End a read that has started and has not ended, before a stop. + + Later steps of the loop end it. It does nothing on a read that has not started + or has ended. + """ + import temporalio.workflow + + if self._seq is None or self._finished: + return + temporalio.workflow._Runtime.current().workflow_close_stream_read( + self, send=self._end is None + ) + self._finished = True + self._records.clear() + if self._waiter is not None and not self._waiter.done(): + self._waiter.set_result(None) + + def _started( + self, + seq: int, + decode: "collections.abc.Callable[[temporalio.api.stream.v1.StreamRecord], StreamRecord[_BodyT]]", + ) -> None: + self._seq = seq + self._decode = decode + + def _take( + self, + records: "collections.abc.Iterable[temporalio.api.stream.v1.StreamRecord]", + end: tuple[StreamReadStop | None, bool, BaseException | None] | None, + ) -> None: + self._records.extend(records) + if end is not None: + self._end = end + if self._waiter is not None and not self._waiter.done(): + self._waiter.set_result(None) + + +def iter_stream( + stream: typing.Any, + *, + timeout: timedelta | None = None, + from_offset: bytes | None = None, + idle_timeout: timedelta | None = None, + max_records: int | None = None, + max_bytes: int | None = None, +) -> StreamReading[typing.Any]: + """Read each Record of a Stream as it arrives, up to a stop. + + It is the ``iter`` of a Workflow Stream. It does no I/O and sends nothing, so it + takes no ``await``. The read starts at the first step of the ``async for`` loop. + Core then reads the Store inside the Workflow Task and gives the Records to the loop + as they arrive. The loop ends at the first stop. Then + :py:attr:`StreamReading.stop` tells which stop it was, and + :py:attr:`StreamReading.done` is true when the Stream is closed and the read gave its + last Record. A new read from :py:attr:`StreamReading.last_offset` loses and repeats + no Record, also after a close or a ``break``. + + .. code-block:: python + + last = None + while True: + reading = tokens.iter(from_offset=last) + async for record in reading: + await shown.append(record.body.text) + last = reading.last_offset + if reading.done: + break + + A read costs one marker in History for each Workflow Task that it spans, whatever + the number of Records. A replay gives the loop the same Records at the same points. + A read has three other costs. + + * A read keeps its Workflow Task open, as a Local Activity does. A Signal, an + Update, a cancel or an Activity result reaches the Workflow at the next Workflow + Task. That is when the read ends, or at the heartbeat of the task at the latest, + which is about 8 s with the default task timeout of 10 s. + * With the default :py:attr:`temporalio.client.StreamReplaySource.STORE`, History + holds no Record. A replay reads the Records from the Store again, so they must + stay in the Store for as long as the Workflow can replay. When they are gone, the + replay fails its Workflow Task with a message that starts with + ``StreamReplayError``. With :py:attr:`temporalio.client.StreamReplaySource.HISTORY`, + History holds the Records, and a replay needs no Store. + * A replay on a Worker with no cache reads the Store again on every Workflow Task. + + ``aclose`` of the read ends it before a stop. A read that the loop leaves with + ``break`` and doesn't close runs on in Core until one of its stops, and its Records + wait in the read. Call ``aclose`` when the code reads no more. + + The read needs a Store that runs on the bridge, as + :py:class:`temporalio.testing.MemoryStreamStore` and the Redis Store do. For a Store + written in Python, the first step of the loop fails the Workflow with a + non-retryable :py:class:`temporalio.exceptions.ApplicationError`. Code in a Query + handler or an Update validator can't read. + + Args: + stream: The Workflow Stream to read. + timeout: The longest time that the read takes, from the time it starts. Its + deadline holds across a replay, a failed Workflow Task and a Worker + restart. ``None`` is no bound. + from_offset: Read the Records after this offset. ``None`` is the start of the + Stream. + idle_timeout: The read ends when no Record comes for this long. ``None`` is 1 s. + Zero gives what is in the Stream and doesn't wait. + max_records: The most Records of the read. ``None`` is no cap. + max_bytes: The most bytes of the Records of the read, each as its encoded + ``StreamRecord``. The first Record always comes. ``None`` is no cap, unless + ``workflow_read_replay_from`` of the Stream config is ``HISTORY``. Then it is + :py:data:`temporalio.common.DEFAULT_STREAM_MAX_BYTES`. + + Returns: + The read, which the loop iterates. + + Raises: + ValueError: ``timeout``, ``max_records`` or ``max_bytes`` is zero or negative, + or ``idle_timeout`` is negative. + """ + if timeout is not None and timeout <= timedelta(): + raise ValueError("timeout must be positive") + if idle_timeout is not None and idle_timeout < timedelta(): + raise ValueError("idle_timeout must not be negative") + if max_records is not None and max_records < 1: + raise ValueError("max_records must be positive") + if max_bytes is not None and max_bytes < 1: + raise ValueError("max_bytes must be positive") + return StreamReading( + stream, + _StreamReadOptions( + from_offset=from_offset, + timeout=timeout, + idle_timeout=idle_timeout, + max_records=max_records, + max_bytes=max_bytes, + ), + ) + + def retry_policy_from_proto( proto: common_pb2.RetryPolicy, ) -> temporalio.common.RetryPolicy: diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index 888c06f29..95c1cce5c 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -16,8 +16,11 @@ ResolveNexusOperationStart, ResolveRequestCancelExternalWorkflow, ResolveSignalExternalWorkflow, + ResolveStreamAppend, + StreamReadEnd, ) from temporalio.bridge.proto.workflow_commands.workflow_commands_pb2 import ( + AppendToStream, CompleteWorkflowExecution, FailWorkflowExecution, ScheduleActivity, @@ -57,6 +60,7 @@ def __init__( skip_search_attributes: bool = False, skip_headers: bool = False, concurrency_limit: int = 1, + skip_stream_bodies: bool = False, ) -> None: """Creates a new command-aware payload visitor. @@ -65,12 +69,35 @@ def __init__( skip_headers: If True, headers are not visited. concurrency_limit: Maximum number of payload visits that may run concurrently during a single call to visit(). Defaults to 1. + skip_stream_bodies: If True, the bodies of a Stream append are not + visited. """ super().__init__( skip_search_attributes=skip_search_attributes, skip_headers=skip_headers, concurrency_limit=concurrency_limit, ) + self.skip_stream_bodies = skip_stream_bodies + + # A Store keeps the bodies of a Workflow's append as the Local Activity path writes + # them, with the codec and with no external storage, so other readers can decode them. + async def _visit_coresdk_workflow_commands_AppendToStream( + self, fs: VisitorFunctions, o: AppendToStream + ) -> None: + if not self.skip_stream_bodies: + await super()._visit_coresdk_workflow_commands_AppendToStream(fs, o) + + # Core builds the failure of a Stream read or append itself, with no codec, so no pass + # may decode it. + async def _visit_coresdk_workflow_activation_StreamReadEnd( + self, fs: VisitorFunctions, o: StreamReadEnd + ) -> None: + pass + + async def _visit_coresdk_workflow_activation_ResolveStreamAppend( + self, fs: VisitorFunctions, o: ResolveStreamAppend + ) -> None: + pass # Workflow commands with payloads async def _visit_coresdk_workflow_commands_CompleteWorkflowExecution( diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index 0365861be..ae7ecc947 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -5,15 +5,24 @@ import asyncio import dataclasses from collections.abc import Generator +from datetime import timedelta from typing import TYPE_CHECKING, Any from nexusrpc import OutputT +import temporalio.api.failure.v1 +import temporalio.api.stream.v1 +import temporalio.bridge.proto.workflow_commands +import temporalio.common import temporalio.exceptions +import temporalio.nexus.system._support import temporalio.workflow from temporalio.client._stream_store import ( _STREAM_OPERATIONS, StreamConfig, + StreamReplaySource, + StreamStore, + _BridgeStreamStore, _request_ref, ) from temporalio.converter import StorageDriverWorkflowInfo @@ -41,7 +50,42 @@ def _no_stream_config_error() -> temporalio.exceptions.ApplicationError: # pyri _ORDERED_OPERATIONS = frozenset({"AppendRecords", "CloseStream"}) -def _bridge_stream_runtimes(config: StreamConfig | None) -> list[tuple[str, Any]]: +def _no_direct_read_error(store: StreamStore) -> temporalio.exceptions.ApplicationError: + # Core reads the Store inside the Workflow Task, and it can't call a Store that runs + # in Python. Each retry of the task would fail the same way for good, so the Workflow + # fails instead, as for a Worker with no Stream config. + return temporalio.exceptions.ApplicationError( + f"The Stream Store '{store.name}' has no Direct read, since it is written in " + "Python. Read the Stream in an Activity, or use a Store that runs on the bridge, " + "such as the Redis Store or MemoryStreamStore.", + type="StreamConfigError", + non_retryable=True, + ) + + +_STREAM_ERROR_DETAILS_TYPE = ( + temporalio.api.stream.v1.StreamErrorDetails.DESCRIPTOR.full_name +) + + +def _stream_error_from_core( # pyright: ignore[reportUnusedFunction] + failure: temporalio.api.failure.v1.Failure, +) -> temporalio.exceptions.StreamError | None: + """Give the Store error of a failure that Core made, or ``None`` for another one.""" + # Core gives a Store error the token of its reason as its type, and puts the reason in + # its details with no codec. The Local Activity path gives the Store's message alone, + # so the token comes off the message. + info = failure.application_failure_info + for payload in info.details.payloads: + if payload.metadata.get("messageType") != _STREAM_ERROR_DETAILS_TYPE.encode(): + continue + details = temporalio.api.stream.v1.StreamErrorDetails.FromString(payload.data) + message = failure.message.removeprefix(f"{info.type}: ") + return temporalio.exceptions._stream_error_from_reason(details.reason, message) + return None + + +def _bridge_stream_runtimes(config: StreamConfig | None) -> list[tuple[str, Any]]: # pyright: ignore[reportUnusedFunction] """Give Core the bridge's object of each Store's Stream runtime, by Store name.""" if config is None: return [] @@ -71,6 +115,42 @@ def from_config(cls, config: StreamConfig | None) -> _WorkflowStreamStore | None """Build it from the Stream config of a client, or give ``None`` when it has none.""" return cls(config=config) if config is not None else None + def read_command( + self, reading: temporalio.nexus.system._support.StreamReading[Any] + ) -> temporalio.bridge.proto.workflow_commands.StartStreamRead: + """Build the start of a Direct read, with no ``seq``.""" + ref = reading._stream.ref + # The choice runs in Workflow code, so a ref of a Store that the config doesn't + # have raises `StreamConfigError` here, with no I/O, and fails the Workflow Task. + store = self.config._store_for(ref, _workflow_target()) + if not isinstance(store, _BridgeStreamStore): + raise _no_direct_read_error(store) + options = reading._options + records_in_history = ( + self.config.workflow_read_replay_from is StreamReplaySource.HISTORY + ) + start = temporalio.bridge.proto.workflow_commands.StartStreamRead( + stream=temporalio.api.stream.v1.StreamRef(id=ref.id, store=store.name), + from_offset=options.from_offset or b"", + records_in_history=records_in_history, + ) + if options.timeout is not None: + start.timeout.FromTimedelta(options.timeout) + if options.idle_timeout is not None: + # Core reads a zero idle timeout as unset, which is 1 s. The least wait that + # Core keeps stands for no wait. + start.idle_timeout.FromTimedelta( + options.idle_timeout or timedelta(milliseconds=1) + ) + if options.max_records is not None: + start.max_records = options.max_records + max_bytes = options.max_bytes + if max_bytes is None and records_in_history: + max_bytes = temporalio.common.DEFAULT_STREAM_MAX_BYTES + if max_bytes is not None: + start.max_bytes = max_bytes + return start + def start_operation( self, instance: _WorkflowInstanceImpl, diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 454a7c2c0..7a8077a08 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -46,6 +46,7 @@ import temporalio.api.common.v1 import temporalio.api.enums.v1 import temporalio.api.sdk.v1 +import temporalio.api.stream.v1 import temporalio.bridge.proto.activity_result import temporalio.bridge.proto.child_workflow import temporalio.bridge.proto.common @@ -58,6 +59,8 @@ import temporalio.converter._serialization_context import temporalio.exceptions import temporalio.nexus.system +import temporalio.nexus.system._support +import temporalio.nexus.system.stream_service import temporalio.workflow from temporalio.client._stream_store import _STREAM_SERVICE from temporalio.converter import StorageDriverStoreContext, StorageDriverWorkflowInfo @@ -95,7 +98,11 @@ WorkflowInboundInterceptor, WorkflowOutboundInterceptor, ) -from ._stream_store import _no_stream_config_error, _WorkflowStreamStore +from ._stream_store import ( + _no_stream_config_error, + _stream_error_from_core, + _WorkflowStreamStore, +) logger = logging.getLogger(__name__) @@ -364,6 +371,12 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._pending_nexus_operations: dict[int, _NexusOperationHandle] = {} self._pending_external_signals: dict[int, tuple[asyncio.Future, str]] = {} self._pending_external_cancels: dict[int, tuple[asyncio.Future, str]] = {} + self._pending_stream_reads: dict[ + int, temporalio.nexus.system._support.StreamReading[Any] + ] = {} + # Reads that Workflow code closed. A job of one of them can still come in the + # activation that the close answers, and is dropped. + self._closed_stream_reads: set[int] = set() # Keyed by type self._curr_seqs: dict[str, int] = {} # TODO(cretz): Any concerns about not sharing this? Maybe the types I @@ -693,6 +706,8 @@ def _apply( self._apply_resolve_signal_external_workflow( job.resolve_signal_external_workflow ) + elif job.HasField("resolve_stream_read"): + self._apply_resolve_stream_read(job.resolve_stream_read) elif job.HasField("signal_workflow"): self._apply_signal_workflow(job.signal_workflow) elif job.HasField("initialize_workflow"): @@ -944,6 +959,39 @@ def _apply_remove_from_cache( for task in self._tasks: task.cancel() + def _apply_resolve_stream_read( + self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamRead + ) -> None: + reading = self._pending_stream_reads.get(job.seq) + if reading is None: + if job.seq in self._closed_stream_reads: + return + raise RuntimeError(f"Failed finding Stream read for sequence {job.seq}") + end: ( + tuple[ + temporalio.nexus.system._support.StreamReadStop | None, + bool, + BaseException | None, + ] + | None + ) = None + if job.HasField("end"): + del self._pending_stream_reads[job.seq] + error: BaseException | None = None + if job.end.HasField("failure"): + error = _stream_error_from_core(job.end.failure) + if error is None: + error = self._context_free_failure_converter.from_failure( + job.end.failure, self._context_free_payload_converter + ) + stop = ( + temporalio.nexus.system._support.StreamReadStop(job.end.stop) + if job.end.stop + else None + ) + end = (stop, job.end.done, error) + reading._take(job.records, end) + def _apply_resolve_activity( self, job: temporalio.bridge.proto.workflow_activation.ResolveActivity ) -> None: @@ -1776,6 +1824,48 @@ async def workflow_start_child_workflow( ) ) + def workflow_start_stream_read( + self, reading: temporalio.nexus.system._support.StreamReading[Any] + ) -> None: + self._assert_not_read_only("start stream read") + store = self._stream_store + if store is None: + raise _no_stream_config_error() + start = store.read_command(reading) + start.seq = self._next_seq("stream_read") + self._add_command().start_stream_read.CopyFrom(start) + # Stream calls have no serialization context, as the system Nexus converters of + # the other Stream calls show. + payload_converter = temporalio.nexus.system._get_payload_converter( + self._context_free_payload_converter, self._context_free_failure_converter + ) + body_type: Any = reading._stream.body_type + record_type = temporalio.nexus.system.stream_service.StreamRecord[body_type] + + def decode( + record: temporalio.api.stream.v1.StreamRecord, + ) -> temporalio.nexus.system.stream_service.StreamRecord[Any]: + return payload_converter._from_transfer_value(record, record_type) + + reading._started(start.seq, decode) + self._pending_stream_reads[start.seq] = reading + + def workflow_close_stream_read( + self, + reading: temporalio.nexus.system._support.StreamReading[Any], + *, + send: bool, + ) -> None: + seq = reading._seq + if seq is None or seq not in self._pending_stream_reads: + return + # An eviction ends every read in Core, so it needs no close. + if send and not self._deleting: + self._assert_not_read_only("close stream read") + self._add_command().close_stream_read.seq = seq + self._closed_stream_reads.add(seq) + del self._pending_stream_reads[seq] + def workflow_start_local_activity( self, activity: Any, diff --git a/temporalio/workflow/__init__.py b/temporalio/workflow/__init__.py index 60b02cc54..4a2eb9708 100644 --- a/temporalio/workflow/__init__.py +++ b/temporalio/workflow/__init__.py @@ -12,6 +12,9 @@ ) # END GENERATED NEXUS SYSTEM EXPORTS +# isort: split +from temporalio.nexus.system._support import StreamReading, StreamReadStop + from ..types import ( AnyType, CallableAsyncNoParam, @@ -332,6 +335,8 @@ "ProtocolReturnType", "ReturnType", "SelfType", + "StreamReading", + "StreamReadStop", # BEGIN GENERATED NEXUS SYSTEM __ALL__ "create_stream", "get_stream", diff --git a/temporalio/workflow/_context.py b/temporalio/workflow/_context.py index a33759c37..a215813d3 100644 --- a/temporalio/workflow/_context.py +++ b/temporalio/workflow/_context.py @@ -500,6 +500,12 @@ async def workflow_start_nexus_operation( event_groups: Sequence[EventGroup] | None = None, ) -> NexusOperationHandle[OutputT]: ... + @abstractmethod + def workflow_start_stream_read(self, reading: Any) -> None: ... + + @abstractmethod + def workflow_close_stream_read(self, reading: Any, *, send: bool) -> None: ... + @abstractmethod def workflow_time_ns(self) -> int: ... From efbe3010f7dd953fb771e3debe2cd3dc14c4353e Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 03:22:01 -0700 Subject: [PATCH 06/59] Made Core append a Workflow's Records inside the Workflow Task. An append of a Workflow Producer on a Store with a Rust runtime goes to Core, which writes it as the Workflow with the task as its epoch. Its tests now expect that Producer and that marker. --- temporalio/worker/_command_aware_visitor.py | 7 +++- temporalio/worker/_stream_store.py | 35 ++++++++++++++++++ temporalio/worker/_workflow_instance.py | 37 +++++++++++++++++++ .../contrib/streams/test_appender_workflow.py | 13 ++++++- tests/nexus/test_stream_store_workflow.py | 34 +++++++++++------ 5 files changed, 111 insertions(+), 15 deletions(-) diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index 95c1cce5c..e6180c1b5 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -81,11 +81,14 @@ def __init__( # A Store keeps the bodies of a Workflow's append as the Local Activity path writes # them, with the codec and with no external storage, so other readers can decode them. + # Each body is one Record, so the codec gets each body alone, as for that path. async def _visit_coresdk_workflow_commands_AppendToStream( self, fs: VisitorFunctions, o: AppendToStream ) -> None: - if not self.skip_stream_bodies: - await super()._visit_coresdk_workflow_commands_AppendToStream(fs, o) + if self.skip_stream_bodies: + return + for body in o.bodies: + await self._visit_temporal_api_common_v1_Payload(fs, body) # Core builds the failure of a Stream read or append itself, with no codec, so no pass # may decode it. diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index ae7ecc947..9a244a42e 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -12,6 +12,7 @@ import temporalio.api.failure.v1 import temporalio.api.stream.v1 +import temporalio.api.streamservice.v1 import temporalio.bridge.proto.workflow_commands import temporalio.common import temporalio.exceptions @@ -171,6 +172,40 @@ def start_operation( # have raises `StreamConfigError` here, with no I/O, and fails the Workflow Task. store = self.config._store_for(ref, _workflow_target()) options = self.config.workflow_call_options + if ( + isinstance(request, temporalio.api.streamservice.v1.AppendRecordsRequest) + and isinstance(store, _BridgeStreamStore) + and request.producer.WhichOneof("kind") == "workflow" + ): + # Core appends inside the Workflow Task and sets the epoch of the Producer + # to that task. A Producer of another kind keeps the Local Activity path. + append = temporalio.bridge.proto.workflow_commands.AppendToStream( + stream=temporalio.api.stream.v1.StreamRef( + id=request.ref.id, store=store.name + ), + bodies=[record.body for record in request.records], + producer=request.producer, + sequence=request.sequence, + ) + append.producer.workflow.ClearField("epoch") + appended = instance._start_stream_append(append) + + async def direct_result() -> OutputT: + response = await appended + return payload_converter._from_transfer_value( + temporalio.api.streamservice.v1.AppendRecordsResponse( + first_offset=response.first_offset, + last_offset=response.last_offset, + ), + input.output_type, + ) + + return _StreamStoreOperationHandle( + instance.create_task( + direct_result(), name=f"stream store: {input.operation_name}" + ) + ) + tails = instance._stream_write_tails tail_key = (store.name, ref.id) previous = ( diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 7a8077a08..45531146e 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -374,6 +374,10 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._pending_stream_reads: dict[ int, temporalio.nexus.system._support.StreamReading[Any] ] = {} + self._pending_stream_appends: dict[ + int, + asyncio.Future[temporalio.bridge.proto.workflow_activation.StreamAppended], + ] = {} # Reads that Workflow code closed. A job of one of them can still come in the # activation that the close answers, and is dropped. self._closed_stream_reads: set[int] = set() @@ -708,6 +712,8 @@ def _apply( ) elif job.HasField("resolve_stream_read"): self._apply_resolve_stream_read(job.resolve_stream_read) + elif job.HasField("resolve_stream_append"): + self._apply_resolve_stream_append(job.resolve_stream_append) elif job.HasField("signal_workflow"): self._apply_signal_workflow(job.signal_workflow) elif job.HasField("initialize_workflow"): @@ -992,6 +998,25 @@ def _apply_resolve_stream_read( end = (stop, job.end.done, error) reading._take(job.records, end) + def _apply_resolve_stream_append( + self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamAppend + ) -> None: + fut = self._pending_stream_appends.pop(job.seq, None) + if fut is None: + raise RuntimeError(f"Failed finding Stream append for sequence {job.seq}") + # Code that stopped waiting for the append has cancelled the future. + if fut.done(): + return + if job.HasField("appended"): + fut.set_result(job.appended) + return + error: BaseException | None = _stream_error_from_core(job.failed) + if error is None: + error = self._context_free_failure_converter.from_failure( + job.failed, self._context_free_payload_converter + ) + fut.set_exception(error) + def _apply_resolve_activity( self, job: temporalio.bridge.proto.workflow_activation.ResolveActivity ) -> None: @@ -1850,6 +1875,18 @@ def decode( reading._started(start.seq, decode) self._pending_stream_reads[start.seq] = reading + def _start_stream_append( + self, append: temporalio.bridge.proto.workflow_commands.AppendToStream + ) -> asyncio.Future[temporalio.bridge.proto.workflow_activation.StreamAppended]: + command = self._add_command() + append.seq = self._next_seq("stream_append") + command.append_to_stream.CopyFrom(append) + fut: asyncio.Future[ + temporalio.bridge.proto.workflow_activation.StreamAppended + ] = self.create_future() + self._pending_stream_appends[append.seq] = fut + return fut + def workflow_close_stream_read( self, reading: temporalio.nexus.system._support.StreamReading[Any], diff --git a/tests/contrib/streams/test_appender_workflow.py b/tests/contrib/streams/test_appender_workflow.py index 998149b2e..7a65e8510 100644 --- a/tests/contrib/streams/test_appender_workflow.py +++ b/tests/contrib/streams/test_appender_workflow.py @@ -5,6 +5,7 @@ from datetime import timedelta from typing import Any +import pytest from google.protobuf.message import Message import temporalio.api.streamservice.v1 @@ -177,12 +178,20 @@ async def test_replay_of_workflow_appender_writes_nothing(client: Client): stream_id = f"stream-{uuid.uuid4()}" handle, _ = await _run(stream_client, InFlightWorkflow, stream_id) - replay_store = BatchStore() + # A replay of an append that Core made needs a Store that runs on the bridge, as the + # first run had. A new memory Store shows any write, since it has no Stream. + replay_store = MemoryStreamStore() await Replayer( workflows=[InFlightWorkflow], stream_config=_with_store(client, replay_store).config()["stream_config"], ).replay_workflow(await handle.fetch_history()) - assert replay_store.operations == [] + with pytest.raises(StreamError) as err: + await ( + _with_store(client, replay_store) + .get_stream(StreamRef(id=stream_id), body_type=str) + .describe() + ) + assert err.value.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND assert await _bodies(stream_client, stream_id) == ["a", "b", "c", "d"] diff --git a/tests/nexus/test_stream_store_workflow.py b/tests/nexus/test_stream_store_workflow.py index 9bb641aa9..53b68da38 100644 --- a/tests/nexus/test_stream_store_workflow.py +++ b/tests/nexus/test_stream_store_workflow.py @@ -104,7 +104,8 @@ async def run(self, stream_id: str) -> StreamRef[str]: # Workflow code appends either with the Workflow `Stream` or with a Local Activity of -# its own, and both write as the Local Activity that runs the append. +# its own. Core makes the first append in the Workflow Task as the Workflow, and the +# second writes as the Local Activity that runs it. _APPEND_WORKFLOWS = pytest.mark.parametrize( "append_workflow", [AppendWorkflow, LocalActivityAppendWorkflow] ) @@ -581,8 +582,12 @@ async def test_workflow_stream_history_has_no_bodies( for event in events if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED ] - # One marker for the create and one for the append. - assert markers == ["core_local_activity", "core_local_activity"] + # One marker for the create and one for the append. Core writes its own marker for + # an append that it makes in the Workflow Task. + append_marker = ( + "core_stream" if append_workflow is AppendWorkflow else "core_local_activity" + ) + assert markers == ["core_local_activity", append_marker] for event in events: for body in _BODIES: assert body.encode() not in event.SerializeToString() @@ -593,13 +598,19 @@ async def test_workflow_stream_replay_does_not_call_store( client: Client, append_workflow: type[Any] ): stream_client = _with_store(client, MemoryStreamStore()) - handle, _ = await _run_append_workflow(stream_client, append_workflow) - replay_store = CountingStore("memory") + handle, ref = await _run_append_workflow(stream_client, append_workflow) + # A replay of an append that Core made needs a Store that runs on the bridge, as the + # first run had. A new memory Store shows any call, since it has no Stream. + replay_store = MemoryStreamStore() await Replayer( workflows=[append_workflow], stream_config=StreamConfig(stores=[replay_store]), ).replay_workflow(await handle.fetch_history()) - assert replay_store.operations == [] + with pytest.raises(StreamError) as err: + await ( + _with_store(client, replay_store).get_stream(ref, body_type=str).describe() + ) + assert err.value.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND async def test_workflow_create_without_stream_config_fails_the_workflow( @@ -785,8 +796,8 @@ async def test_workflow_task_retry_appends_with_higher_epoch( task_queue=worker.task_queue, ) records = await _read_all(stream_client, stream_id) - # The failed Workflow Task records no marker, so its retry runs the Local Activity - # again and writes a second copy. + # The failed Workflow Task records no marker, so its retry appends again and writes + # a second copy. assert [record.body for record in records] == [*_BODIES, *_BODIES] producers = _producers(records) assert len({producer.id for producer in producers}) == 1 @@ -1219,9 +1230,10 @@ async def test_activity_interceptors_see_the_close_activity(client: Client): id=f"workflow-{uuid.uuid4()}", task_queue=worker.task_queue, ) - # The create, the append and the close each run as a Local Activity. - assert interceptor.activity_types == ["__temporal_stream_store_call:memory"] * 3 - assert interceptor.is_local == [True] * 3 + # The create and the close each run as a Local Activity. Core makes the append in + # the Workflow Task, with no Activity. + assert interceptor.activity_types == ["__temporal_stream_store_call:memory"] * 2 + assert interceptor.is_local == [True] * 2 class SlowFirstAppendStore(LostAnswerStore): From c2655d174e7b1f068f3bf02ca3d2bc60d4ad6a91 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 03:36:02 -0700 Subject: [PATCH 07/59] Let a replay with no Stream config start a read that History holds. A marker with the Records needs no Store, so a Replayer with no Stream config can replay it. The read takes its Store from the ref, as the first run had it. --- temporalio/worker/_stream_store.py | 66 +++++++++++++++---------- temporalio/worker/_workflow_instance.py | 12 ++++- 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index 9a244a42e..c63d52945 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -86,6 +86,39 @@ def _stream_error_from_core( # pyright: ignore[reportUnusedFunction] return None +def _read_command( # pyright: ignore[reportUnusedFunction] + reading: temporalio.nexus.system._support.StreamReading[Any], + store_name: str, + *, + records_in_history: bool, +) -> temporalio.bridge.proto.workflow_commands.StartStreamRead: + """Build the start of a Direct read on the Store ``store_name``, with no ``seq``.""" + options = reading._options + start = temporalio.bridge.proto.workflow_commands.StartStreamRead( + stream=temporalio.api.stream.v1.StreamRef( + id=reading._stream.ref.id, store=store_name + ), + from_offset=options.from_offset or b"", + records_in_history=records_in_history, + ) + if options.timeout is not None: + start.timeout.FromTimedelta(options.timeout) + if options.idle_timeout is not None: + # Core reads a zero idle timeout as unset, which is 1 s. The least wait that Core + # keeps stands for no wait. + start.idle_timeout.FromTimedelta( + options.idle_timeout or timedelta(milliseconds=1) + ) + if options.max_records is not None: + start.max_records = options.max_records + max_bytes = options.max_bytes + if max_bytes is None and records_in_history: + max_bytes = temporalio.common.DEFAULT_STREAM_MAX_BYTES + if max_bytes is not None: + start.max_bytes = max_bytes + return start + + def _bridge_stream_runtimes(config: StreamConfig | None) -> list[tuple[str, Any]]: # pyright: ignore[reportUnusedFunction] """Give Core the bridge's object of each Store's Stream runtime, by Store name.""" if config is None: @@ -120,37 +153,18 @@ def read_command( self, reading: temporalio.nexus.system._support.StreamReading[Any] ) -> temporalio.bridge.proto.workflow_commands.StartStreamRead: """Build the start of a Direct read, with no ``seq``.""" - ref = reading._stream.ref # The choice runs in Workflow code, so a ref of a Store that the config doesn't # have raises `StreamConfigError` here, with no I/O, and fails the Workflow Task. - store = self.config._store_for(ref, _workflow_target()) + store = self.config._store_for(reading._stream.ref, _workflow_target()) if not isinstance(store, _BridgeStreamStore): raise _no_direct_read_error(store) - options = reading._options - records_in_history = ( - self.config.workflow_read_replay_from is StreamReplaySource.HISTORY - ) - start = temporalio.bridge.proto.workflow_commands.StartStreamRead( - stream=temporalio.api.stream.v1.StreamRef(id=ref.id, store=store.name), - from_offset=options.from_offset or b"", - records_in_history=records_in_history, + return _read_command( + reading, + store.name, + records_in_history=( + self.config.workflow_read_replay_from is StreamReplaySource.HISTORY + ), ) - if options.timeout is not None: - start.timeout.FromTimedelta(options.timeout) - if options.idle_timeout is not None: - # Core reads a zero idle timeout as unset, which is 1 s. The least wait that - # Core keeps stands for no wait. - start.idle_timeout.FromTimedelta( - options.idle_timeout or timedelta(milliseconds=1) - ) - if options.max_records is not None: - start.max_records = options.max_records - max_bytes = options.max_bytes - if max_bytes is None and records_in_history: - max_bytes = temporalio.common.DEFAULT_STREAM_MAX_BYTES - if max_bytes is not None: - start.max_bytes = max_bytes - return start def start_operation( self, diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 45531146e..a1ce88901 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -100,6 +100,7 @@ ) from ._stream_store import ( _no_stream_config_error, + _read_command, _stream_error_from_core, _WorkflowStreamStore, ) @@ -1854,9 +1855,16 @@ def workflow_start_stream_read( ) -> None: self._assert_not_read_only("start stream read") store = self._stream_store - if store is None: + if store is not None: + start = store.read_command(reading) + elif self._is_replaying: + # A replay of a read whose marker holds its Records reads no Store, so it needs + # no Stream config. The ref names the Store, as the first run had it. + start = _read_command( + reading, reading._stream.ref.store or "", records_in_history=True + ) + else: raise _no_stream_config_error() - start = store.read_command(reading) start.seq = self._next_seq("stream_read") self._add_command().start_stream_read.CopyFrom(start) # Stream calls have no serialization context, as the system Nexus converters of From 8198e0df53c4798249c83f371f678c839b38c096 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 03:36:21 -0700 Subject: [PATCH 08/59] Tested the Direct reads and appends of a Stream in Workflow code. The tests cover each stop, the marker, replays, a Worker with no cache, a failed task, a Signal, a Store written in Python and a payload codec, and measure a read of 200 Records. --- tests/nexus/test_stream_direct.py | 948 ++++++++++++++++++++++++++++++ 1 file changed, 948 insertions(+) create mode 100644 tests/nexus/test_stream_direct.py diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py new file mode 100644 index 000000000..ba7480a36 --- /dev/null +++ b/tests/nexus/test_stream_direct.py @@ -0,0 +1,948 @@ +"""Reads and appends of a Stream that Core makes inside the Workflow Task.""" + +from __future__ import annotations + +import asyncio +import uuid +from collections.abc import AsyncGenerator +from dataclasses import dataclass +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import activity, workflow +from temporalio.api.enums.v1 import EventType, StreamErrorReason +from temporalio.client import ( + Client, + StreamConfig, + StreamReplaySource, + WorkflowFailureError, + WorkflowHandle, + WorkflowHistory, +) +from temporalio.common import RawValue +from temporalio.converter import DataConverter +from temporalio.exceptions import ApplicationError, StreamError +from temporalio.nexus.system.stream_service import ( + Producer, + ProducerKindWorkflow, + StreamRecord, + StreamRef, +) +from temporalio.testing import MemoryStreamStore +from temporalio.worker import Replayer +from tests.helpers import new_worker +from tests.helpers.streams import DictStreamStore +from tests.test_converter import SimpleCodec + +# A Record body that a test can look for in the bytes of History. +_MARK = "direct-read-mark" + + +def _stream_client( + client: Client, + store: Any = None, + *, + replay_from: StreamReplaySource = StreamReplaySource.STORE, + codec: bool = False, +) -> Client: + config = client.config() + config["stream_config"] = StreamConfig( + stores=[store if store is not None else MemoryStreamStore()], + workflow_read_replay_from=replay_from, + ) + if codec: + config["data_converter"] = DataConverter(payload_codec=SimpleCodec()) + return Client(**config) + + +def _markers(history: WorkflowHistory) -> list[str]: + return [ + event.marker_recorded_event_attributes.marker_name + for event in history.events + if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + ] + + +def _stream_markers_per_task(history: WorkflowHistory) -> list[int]: + """Give the number of `core_stream` markers that each completed Workflow Task wrote.""" + counts: list[int] = [] + for event in history.events: + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED: + counts.append(0) + elif ( + event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + and event.marker_recorded_event_attributes.marker_name == "core_stream" + ): + counts[-1] += 1 + return counts + + +def _completed_tasks(history: WorkflowHistory) -> int: + return sum( + 1 + for event in history.events + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED + ) + + +async def _filled_stream( + client: Client, bodies: list[str], *, close: bool = True +) -> StreamRef[str]: + stream = await client.create_stream(id=f"stream-{uuid.uuid4()}", body_type=str) + if bodies: + await stream.append(*bodies) + if close: + await stream.close() + return stream.ref + + +async def _read_all(client: Client, ref: StreamRef[str]) -> list[StreamRecord[str]]: + stream = client.get_stream(ref, body_type=str) + return [record async for record in stream.iter_read_items()] + + +async def _run( + client: Client, + workflow_type: type[Any], + *args: Any, + activities: list[Any] | None = None, + max_cached_workflows: int = 1000, + task_timeout: timedelta | None = None, +) -> tuple[WorkflowHandle[Any, Any], Any]: + async with new_worker( + client, + workflow_type, + activities=activities or [], + max_cached_workflows=max_cached_workflows, + ) as worker: + handle: WorkflowHandle[Any, Any] = await client.start_workflow( + workflow_type.run, + args=list(args), + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + task_timeout=task_timeout, + ) + return handle, await handle.result() + + +# A generator over `iter` that reads the Stream of an Activity that still runs. + + +@dataclass +class ModelCall: + chunks_id: str + seen_id: str + + +@activity.defn +async def stream_model_output(call: ModelCall) -> None: + client = activity.client() + chunks = client.get_stream(StreamRef(id=call.chunks_id), body_type=str) + seen = client.get_stream(StreamRef(id=call.seen_id), body_type=str) + await chunks.append("Hel", "lo") + # The Activity writes the rest only after the Workflow says that it has the first + # chunks. So the test passes only if Workflow code gets chunks while the Activity runs. + told = False + async for _ in seen.iter_read_items(wait=timedelta(seconds=30)): + told = True + break + if not told: + raise ApplicationError( + "The Workflow got no chunk while the Activity ran", non_retryable=True + ) + await chunks.append(", ", "world") + await chunks.close() + + +async def model_chunks(chunks_id: str) -> AsyncGenerator[str, None]: + """Yield each chunk of the model's Stream, as the model stub of a library does.""" + stream = workflow.get_stream(StreamRef(id=chunks_id), body_type=str) + last = None + while True: + reading = stream.iter(from_offset=last) + async for record in reading: + yield record.body + last = reading.last_offset + if reading.done: + return + + +@workflow.defn +class LiveModelWorkflow: + @workflow.run + async def run(self, call: ModelCall) -> str: + model = workflow.start_activity( + stream_model_output, call, start_to_close_timeout=timedelta(seconds=60) + ) + seen = workflow.get_stream(StreamRef(id=call.seen_id), body_type=str) + text = "" + async for chunk in model_chunks(call.chunks_id): + text += chunk + if text == "Hello": + await seen.append("the first chunks") + await model + return text + + +async def test_generator_over_iter_reads_a_running_activity(client: Client): + stream_client = _stream_client(client) + chunks = await _filled_stream(stream_client, [], close=False) + seen = await _filled_stream(stream_client, [], close=False) + handle, text = await _run( + stream_client, + LiveModelWorkflow, + ModelCall(chunks.id, seen.id), + activities=[stream_model_output], + ) + assert text == "Hello, world" + history = await handle.fetch_history() + # The reads and the append write no Local Activity marker, and at most one marker for + # each Workflow Task. + assert "core_local_activity" not in _markers(history) + assert "core_stream" in _markers(history) + assert max(_stream_markers_per_task(history)) == 1 + + +# One read and its stop. + + +@dataclass +class ReadArgs: + stream_id: str + from_offset: str | None = None + timeout_ms: int | None = None + idle_timeout_ms: int | None = None + max_records: int | None = None + max_bytes: int | None = None + store: str = "memory" + + +@dataclass +class ReadResult: + bodies: list[str] + offsets: list[str] + last_offset: str | None + stop: str | None + done: bool + + +def _ms(value: int | None) -> timedelta | None: + return None if value is None else timedelta(milliseconds=value) + + +def _hex(value: bytes | None) -> str | None: + return None if value is None else value.hex() + + +@workflow.defn +class ReadOnceWorkflow: + @workflow.run + async def run(self, args: ReadArgs) -> ReadResult: + # The ref names its Store, so a replay with no Stream config knows it. + ref = StreamRef[str](id=args.stream_id, store=args.store) + stream = workflow.get_stream(ref, body_type=str) + reading = stream.iter( + timeout=_ms(args.timeout_ms), + from_offset=( + bytes.fromhex(args.from_offset) + if args.from_offset is not None + else None + ), + idle_timeout=_ms(args.idle_timeout_ms), + max_records=args.max_records, + max_bytes=args.max_bytes, + ) + assert reading.stop is None and not reading.done + records = [record async for record in reading] + return ReadResult( + bodies=[record.body for record in records], + offsets=[record.offset.hex() for record in records], + last_offset=_hex(reading.last_offset), + stop=reading.stop.name if reading.stop is not None else None, + done=reading.done, + ) + + +async def _read_once(client: Client, args: ReadArgs) -> ReadResult: + _, result = await _run(client, ReadOnceWorkflow, args) + return result + + +async def test_read_stops_at_done(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + result = await _read_once(stream_client, ReadArgs(ref.id)) + assert result.bodies == ["a", "b", "c"] + assert (result.stop, result.done) == ("Done", True) + assert result.last_offset == result.offsets[-1] + + +async def test_read_stops_at_idle(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b"], close=False) + result = await _read_once(stream_client, ReadArgs(ref.id, idle_timeout_ms=200)) + assert result.bodies == ["a", "b"] + assert (result.stop, result.done) == ("Idle", False) + assert result.last_offset == result.offsets[-1] + + +async def test_read_stops_at_timeout(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + result = await _read_once( + stream_client, ReadArgs(ref.id, timeout_ms=500, idle_timeout_ms=30_000) + ) + assert result.bodies == ["a"] + assert (result.stop, result.done) == ("Timeout", False) + + +async def test_read_stops_at_max_records(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c", "d"]) + result = await _read_once(stream_client, ReadArgs(ref.id, max_records=3)) + assert result.bodies == ["a", "b", "c"] + assert (result.stop, result.done) == ("MaxRecords", False) + # The next read starts after the last Record that this one gave. + rest = await _read_once( + stream_client, ReadArgs(ref.id, from_offset=result.last_offset) + ) + assert rest.bodies == ["d"] + assert (rest.stop, rest.done) == ("Done", True) + + +async def test_read_stops_at_max_bytes(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + # The first Record always comes, even over the cap. + result = await _read_once(stream_client, ReadArgs(ref.id, max_bytes=1)) + assert result.bodies == ["a"] + assert (result.stop, result.done) == ("MaxBytes", False) + + +async def test_read_with_zero_idle_timeout_gives_what_is_there(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b"], close=False) + handle, result = await _run( + stream_client, ReadOnceWorkflow, ReadArgs(ref.id, idle_timeout_ms=0) + ) + assert result.bodies == ["a", "b"] + assert (result.stop, result.done) == ("Idle", False) + events = (await handle.fetch_history()).events + # The read waits for no new Record, so the run takes far less than the default 1 s. + took = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert took < timedelta(milliseconds=800) + + +# What `iter` checks and when the read starts. + + +@workflow.defn +class IterArgumentsWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + errors: list[str] = [] + cases: list[dict[str, Any]] = [ + {"timeout": timedelta()}, + {"timeout": timedelta(seconds=-1)}, + {"idle_timeout": timedelta(seconds=-1)}, + {"max_records": 0}, + {"max_bytes": -1}, + ] + for arguments in cases: + try: + stream.iter(**arguments) + except ValueError as err: + errors.append(str(err)) + # `iter` sends nothing, so a read that the code never steps writes no marker. + stream.iter(idle_timeout=timedelta()) + return errors + + +async def test_iter_checks_its_arguments_and_sends_nothing(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"]) + handle, errors = await _run(stream_client, IterArgumentsWorkflow, ref) + assert errors == [ + "timeout must be positive", + "timeout must be positive", + "idle_timeout must not be negative", + "max_records must be positive", + "max_bytes must be positive", + ] + assert _markers(await handle.fetch_history()) == [] + + +@workflow.defn +class QueryReadWorkflow: + def __init__(self) -> None: + self.ref: StreamRef[str] | None = None + + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + self.ref = ref + await workflow.wait_condition(lambda: False) + + @workflow.query + def start_read(self) -> str: + assert self.ref is not None + reading = workflow.get_stream(self.ref, body_type=str).iter() + step = reading.__anext__() + try: + step.send(None) + except Exception as err: + return type(err).__name__ + finally: + step.close() + return "started" + + +async def test_read_in_a_query_raises_the_read_only_error(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"]) + async with new_worker(stream_client, QueryReadWorkflow) as worker: + handle = await stream_client.start_workflow( + QueryReadWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await handle.query(QueryReadWorkflow.start_read) == ( + "ReadOnlyContextError" + ) + await handle.terminate() + + +# A loop of reads from `last_offset`. + + +@workflow.defn +class LoopReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[Any]: + stream = workflow.get_stream(ref, body_type=str) + got: list[str] = [] + # The first read leaves the loop with `break` and no close. + first = stream.iter() + async for record in first: + got.append(record.body) + if len(got) == 3: + break + # The second read closes in the middle. + second = stream.iter(from_offset=first.last_offset) + async for record in second: + got.append(record.body) + if len(got) == 5: + await second.aclose() + await second.aclose() + after_close = [record.body async for record in second] + closed = [second.stop, second.done, after_close] + # The last reads go on from there to the end. + last = second.last_offset + while True: + reading = stream.iter(from_offset=last, max_records=2) + async for record in reading: + got.append(record.body) + last = reading.last_offset + if reading.done: + break + return [got, closed] + + +async def test_loop_from_last_offset_gets_each_record_once(client: Client): + stream_client = _stream_client(client) + bodies = [f"record-{index}" for index in range(10)] + ref = await _filled_stream(stream_client, bodies) + _, (got, closed) = await _run(stream_client, LoopReadWorkflow, ref) + assert got == bodies + assert closed == [None, False, []] + + +# The marker in History. + + +async def _history_of_a_read( + client: Client, replay_from: StreamReplaySource +) -> tuple[WorkflowHistory, StreamRef[str], Client]: + stream_client = _stream_client(client, replay_from=replay_from) + ref = await _filled_stream(stream_client, [f"{_MARK}-1", f"{_MARK}-2"]) + handle, result = await _run(stream_client, ReadOnceWorkflow, ReadArgs(ref.id)) + assert result.bodies == [f"{_MARK}-1", f"{_MARK}-2"] + return await handle.fetch_history(), ref, stream_client + + +def _markers_have_mark(history: WorkflowHistory) -> bool: + # The result of the Workflow holds the bodies, so only the markers count. + return any( + _MARK.encode() in event.SerializeToString() + for event in history.events + if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + ) + + +async def test_store_marker_holds_no_body(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + assert "core_stream" in _markers(history) + assert not _markers_have_mark(history) + + +async def test_history_marker_holds_the_bodies(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.HISTORY) + assert "core_stream" in _markers(history) + assert _markers_have_mark(history) + + +# Replay. + + +async def test_replay_of_store_marker_reads_the_store(client: Client): + history, _, stream_client = await _history_of_a_read( + client, StreamReplaySource.STORE + ) + config = stream_client.config()["stream_config"] + await Replayer(workflows=[ReadOnceWorkflow], stream_config=config).replay_workflow( + history + ) + + +async def _failed_replay(history: WorkflowHistory, config: StreamConfig | None) -> str: + # Core fails the Workflow Task of the replay before any activation. The bound keeps a + # replay that never ends from holding the test. + with pytest.raises(Exception) as err: + await asyncio.wait_for( + Replayer( + workflows=[ReadOnceWorkflow], stream_config=config + ).replay_workflow(history), + timeout=10, + ) + return f"{type(err.value).__name__}: {err.value}" + + +async def test_replay_of_store_marker_with_no_stream_config_fails(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + assert "StreamReplayError" in await _failed_replay(history, None) + + +async def test_replay_of_store_marker_fails_when_records_are_gone(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + # A new memory Store of the same name has none of the Records. + message = await _failed_replay(history, StreamConfig(stores=[MemoryStreamStore()])) + assert "StreamReplayError" in message + + +async def test_replay_of_history_marker_needs_no_stream_config(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.HISTORY) + await Replayer(workflows=[ReadOnceWorkflow]).replay_workflow(history) + + +# Two reads at once, on a Worker with no cache and on a cached one. + + +@activity.defn +async def tag(body: str) -> str: + return f"tagged {body}" + + +@dataclass +class TwoReadsArgs: + first_id: str + second_id: str + out_id: str + + +@dataclass +class TwoReadsResult: + first: list[str] + second: list[str] + count: int + reads: int + + +@workflow.defn +class TwoReadsWorkflow: + def __init__(self) -> None: + self.count = 0 + self.reads = 0 + + async def _read(self, stream_id: str, out: Any) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got: list[str] = [] + last = None + while True: + self.reads += 1 + reading = stream.iter(from_offset=last, max_records=2) + async for record in reading: + tagged = await workflow.execute_local_activity( + tag, record.body, start_to_close_timeout=timedelta(seconds=10) + ) + await out.append(tagged) + self.count += 1 + got.append(tagged) + last = reading.last_offset + if reading.done: + return got + # A timer ends the Workflow Task, so a Worker with no cache replays each read. + await workflow.sleep(0.05) + + @workflow.run + async def run(self, args: TwoReadsArgs) -> TwoReadsResult: + out = workflow.get_stream(StreamRef(id=args.out_id), body_type=str) + first, second = await asyncio.gather( + self._read(args.first_id, out), self._read(args.second_id, out) + ) + return TwoReadsResult(first, second, self.count, self.reads) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_two_reads_with_and_without_a_cache( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + first = [f"first-{index}" for index in range(5)] + second = [f"second-{index}" for index in range(5)] + out_ref = await _filled_stream(stream_client, [], close=False) + args = TwoReadsArgs( + first_id=(await _filled_stream(stream_client, first)).id, + second_id=(await _filled_stream(stream_client, second)).id, + out_id=out_ref.id, + ) + handle, result = await _run( + stream_client, + TwoReadsWorkflow, + args, + activities=[tag], + max_cached_workflows=max_cached_workflows, + ) + assert result == TwoReadsResult( + first=[f"tagged {body}" for body in first], + second=[f"tagged {body}" for body in second], + count=10, + reads=6, + ) + # Each Record went to the out Stream once, whatever the replays. + out = [record.body for record in await _read_all(stream_client, out_ref)] + assert sorted(out) == sorted(result.first + result.second) + assert _completed_tasks(await handle.fetch_history()) > 2 + + +# A Direct append. + + +@workflow.defn +class AppendWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str | None]: + stream = workflow.get_stream(ref, body_type=str) + appended = await stream.append("a", "b") + # A timer ends the Workflow Task, so a Worker with no cache replays the append. + await workflow.sleep(0.05) + return [_hex(appended.first_offset), _hex(appended.last_offset)] + + +async def test_direct_append(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, offsets = await _run( + stream_client, AppendWorkflow, ref, max_cached_workflows=0 + ) + records = await _read_all(stream_client, ref) + # The replay of the next Workflow Task appended nothing. + assert [record.body for record in records] == ["a", "b"] + assert offsets == [record.offset.hex() for record in records] + history = await handle.fetch_history() + run_id = handle.result_run_id + started = [ + event.event_id + for event in history.events + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_STARTED + ] + for record in records: + producer = record.producer + assert isinstance(producer, Producer) + assert producer.id == f"wf:{run_id}" + assert isinstance(producer.kind, ProducerKindWorkflow) + assert producer.kind.value.run_id == run_id + epoch = producer.kind.value.epoch + assert epoch is not None + # The epoch is the first Workflow Task, which made the append. + assert (epoch.started_event_id, epoch.attempt) == (started[0], 1) + assert _markers(history) == ["core_stream"] + # A Replayer appends nothing either. + await Replayer( + workflows=[AppendWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(history) + assert len(await _read_all(stream_client, ref)) == 2 + + +@workflow.defn +class RefusedAppendWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + try: + await stream.append("a") + except StreamError as err: + # A timer only on the error path, so a replay without the error differs. + await workflow.sleep(0.01) + return [ + StreamErrorReason.Name(err.reason), + err.message, + str(err.non_retryable), + ] + return ["appended"] + + +async def test_refused_append_raises_stream_error_live_and_in_replay(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"]) + handle, result = await _run(stream_client, RefusedAppendWorkflow, ref) + # The client path gives the same error for the same append. + with pytest.raises(StreamError) as client_err: + await stream_client.get_stream(ref, body_type=str).append("a") + assert result == [ + "STREAM_ERROR_REASON_CLOSED", + client_err.value.message, + "True", + ] + await Replayer( + workflows=[RefusedAppendWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +# A Workflow Task that fails once in the middle of a read. + +_fail_read_task = True + + +@workflow.defn(sandboxed=False) +class FailOnceInReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str], out_ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + out = workflow.get_stream(out_ref, body_type=str) + got: list[str] = [] + async for record in stream.iter(): + got.append(record.body) + if record.body == "record-2": + await out.append("seen record-2") + global _fail_read_task + if _fail_read_task: + _fail_read_task = False + raise RuntimeError("Failed the Workflow Task once") + return got + + +async def test_failed_task_in_a_read_gets_each_record_once(client: Client): + global _fail_read_task + _fail_read_task = True + stream_client = _stream_client(client) + bodies = [f"record-{index}" for index in range(5)] + ref = await _filled_stream(stream_client, bodies) + out_ref = await _filled_stream(stream_client, [], close=False) + _, got = await _run(stream_client, FailOnceInReadWorkflow, ref, out_ref) + assert not _fail_read_task + assert got == bodies + # The append of the failed task reached the Store, and its retry appends again. + out = await _read_all(stream_client, out_ref) + assert [record.body for record in out] == ["seen record-2"] * 2 + epochs = [] + for record in out: + assert record.producer is not None + assert isinstance(record.producer.kind, ProducerKindWorkflow) + epoch = record.producer.kind.value.epoch + assert epoch is not None + epochs.append((epoch.started_event_id, epoch.attempt)) + assert epochs[0] < epochs[1] + + +# A Signal while a read runs. + + +@workflow.defn +class SignalCloseWorkflow: + def __init__(self) -> None: + self.reading: Any = None + self.signaled = False + + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[Any]: + stream = workflow.get_stream(ref, body_type=str) + self.reading = stream.iter(idle_timeout=timedelta(seconds=60)) + got = [record.body async for record in self.reading] + return [got, self.signaled, self.reading.stop] + + @workflow.signal + async def stop(self) -> None: + self.signaled = True + await self.reading.aclose() + + +async def test_signal_reaches_a_running_read_and_closes_it(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, SignalCloseWorkflow) as worker: + handle = await stream_client.start_workflow( + SignalCloseWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + # The heartbeat of an open task comes at 80 % of its timeout. + task_timeout=timedelta(seconds=2), + ) + await asyncio.sleep(0.5) + await handle.signal(SignalCloseWorkflow.stop) + result = await asyncio.wait_for(handle.result(), timeout=30) + assert result == [["a"], True, None] + + +# A Store written in Python. + + +@workflow.defn +class PythonStoreAppendWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + await workflow.get_stream(ref, body_type=str).append("a") + + +async def test_python_store_appends_through_its_local_activity(client: Client): + stream_client = _stream_client(client, DictStreamStore()) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run(stream_client, PythonStoreAppendWorkflow, ref) + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + assert _markers(await handle.fetch_history()) == ["core_local_activity"] + + +async def test_python_store_refuses_a_read(client: Client): + stream_client = _stream_client(client, DictStreamStore()) + ref = await _filled_stream(stream_client, ["a"]) + with pytest.raises(WorkflowFailureError) as err: + await _run(stream_client, ReadOnceWorkflow, ReadArgs(ref.id, store="dict")) + cause = err.value.cause + assert isinstance(cause, ApplicationError) + assert cause.type == "StreamConfigError" + assert cause.non_retryable + assert cause.message == ( + "The Stream Store 'dict' has no Direct read, since it is written in Python. " + "Read the Stream in an Activity, or use a Store that runs on the bridge, such " + "as the Redis Store or MemoryStreamStore." + ) + + +# A payload codec. + + +@workflow.defn +class AppendThenReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + await stream.append("a", "b") + return [record.body async for record in stream.iter(idle_timeout=timedelta())] + + +async def test_payload_codec_covers_direct_append_and_read(client: Client): + store = MemoryStreamStore() + stream_client = _stream_client(client, store, codec=True) + ref = await _filled_stream(stream_client, [], close=False) + _, bodies = await _run(stream_client, AppendThenReadWorkflow, ref) + assert bodies == ["a", "b"] + # A client without the codec reads the bodies as the Store keeps them. + raw = _stream_client(client, store).get_stream( + StreamRef(id=ref.id), body_type=RawValue + ) + stored = [record.body.payload async for record in raw.iter_read_items()] + assert [body.metadata["simple-codec"] for body in stored] == [b"true", b"true"] + + +# A close after a Direct append. + + +@workflow.defn +class AppendThenCloseWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + stream = workflow.get_stream(ref, body_type=str) + await stream.append("a") + await stream.close() + + +async def test_close_after_direct_append(client: Client): + # The append writes as the Workflow with its task as the epoch. The close, from an + # Activity, sends the same Workflow Producer with no epoch, and the Store fences it. + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + await _run(stream_client, AppendThenCloseWorkflow, ref) + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + + +# A measurement. + + +@dataclass +class Burst: + stream_id: str + records: int = 200 + batches: int = 20 + over_ms: int = 2000 + + +@activity.defn +async def write_burst(burst: Burst) -> None: + stream = activity.client().get_stream(StreamRef(id=burst.stream_id), body_type=str) + size = burst.records // burst.batches + for batch in range(burst.batches): + await stream.append( + *(f"record-{batch * size + index}" for index in range(size)) + ) + await asyncio.sleep(burst.over_ms / 1000 / burst.batches) + await stream.close() + + +@workflow.defn +class MeasureWorkflow: + @workflow.run + async def run(self, burst: Burst) -> int: + writer = workflow.start_activity( + write_burst, burst, start_to_close_timeout=timedelta(seconds=60) + ) + stream = workflow.get_stream(StreamRef(id=burst.stream_id), body_type=str) + count = 0 + last = None + while True: + reading = stream.iter(from_offset=last) + async for _ in reading: + count += 1 + last = reading.last_offset + if reading.done: + break + await writer + return count + + +@pytest.mark.parametrize( + "replay_from", [StreamReplaySource.STORE, StreamReplaySource.HISTORY] +) +async def test_measure_read_of_200_records_over_2_seconds( + client: Client, replay_from: StreamReplaySource +): + stream_client = _stream_client(client, replay_from=replay_from) + ref = await _filled_stream(stream_client, [], close=False) + handle, count = await _run( + stream_client, MeasureWorkflow, Burst(ref.id), activities=[write_burst] + ) + assert count == 200 + history = await handle.fetch_history() + sizes = [ + event.marker_recorded_event_attributes.ByteSize() + for event in history.events + if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + ] + print( + f"\n{replay_from.name}: {_completed_tasks(history)} Workflow Tasks, " + f"{len(sizes)} markers, {sum(sizes)} marker bytes, sizes {sizes}" + ) From 5e2ef66dd7a409dd7a715e631cdc436d769d91f9 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 03:38:31 -0700 Subject: [PATCH 09/59] Documented the Direct reads and appends and their costs. A read keeps its Workflow Task open, and a replay of a `STORE` read needs the Records in the Store. The README, the changelog and the docs of the Store and the Replayer now say so. --- changelog/added/patient-river-heron.md | 2 ++ temporalio/client/_stream_store.py | 11 ++++-- temporalio/contrib/streams/README.md | 44 +++++++++++++++++++++++ temporalio/exceptions.py | 7 ++-- temporalio/worker/_replayer.py | 12 +++++-- tests/nexus/test_stream_store_workflow.py | 4 +-- 6 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 changelog/added/patient-river-heron.md diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md new file mode 100644 index 000000000..0c31d9fbe --- /dev/null +++ b/changelog/added/patient-river-heron.md @@ -0,0 +1,2 @@ +Add `iter` to the Workflow `Stream`, so Workflow code reads each Record of a Stream as it arrives. Core reads the Store inside the Workflow Task, and a read costs one marker in History for each Workflow Task that it spans. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). +A Workflow's append on the Redis Store or `MemoryStreamStore` now runs in Core inside the Workflow Task, with no Local Activity. It writes as the Workflow, with its Workflow Task as the epoch. diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index 76913a435..6687edb0f 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -65,6 +65,8 @@ class StreamStoreCallOptions: Workflow code can't do I/O, so the default :py:meth:`StreamStore.call_from_workflow` calls its Store from a Local Activity. + Core makes a read and a Workflow's append on a Store with a Rust runtime itself, with + no Activity, so these options don't apply to them. A :py:class:`StreamConfig` holds them in :py:attr:`StreamConfig.workflow_call_options`, and every Store of that config @@ -191,6 +193,11 @@ async def call_from_workflow( task fails after a close, the Stream stays closed. The retry of the task then can't append to it. + On a Store with a Rust runtime, such as the Redis Store and + :py:class:`temporalio.testing.MemoryStreamStore`, Core appends for Workflow code + inside the Workflow Task, and this method gets no append of a Producer of kind + Workflow. + Args: operation: The Nexus operation name, ``CreateStream``, ``AppendRecords``, ``CloseStream`` or ``DescribeStream`` in this release. @@ -352,8 +359,8 @@ def check_response( temporalio.api.streamservice.v1.ReadRecordsRequest, temporalio.api.streamservice.v1.ReadRecordsResponse, workflow_refusal=( - "Workflow code can't read a Stream on a Store in this release. " - "Call an Activity that reads it through its client." + "Workflow code reads a Stream with `iter` of the Workflow Stream, not " + "with the ReadRecords operation." ), ), "CloseStream": _StreamOperation( diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 2e013f666..06c2e8f93 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -42,3 +42,47 @@ async def write_parts(ref: StreamRef[str], parts: list[str]) -> None: The retry's Records continue the first attempt's epoch, so a reader does not take them as a replacement of the first attempt's Records. + +## Reads and appends in Workflow code + +The helpers run on the Streams API, so they follow how Workflow code reads and appends. On a +Store with a Rust runtime, such as the Redis Store or `MemoryStreamStore`, Core reads and +appends a Stream itself, inside the Workflow Task. A read is `iter` of the Workflow `Stream`. +It gives each Record as it arrives and ends at a stop. + +```python +tokens = workflow.get_stream(tokens_ref, body_type=Token) +shown = workflow.get_stream(shown_ref, body_type=str) +last = None +while True: + reading = tokens.iter(from_offset=last) + async for record in reading: + await shown.append(record.body.text) + last = reading.last_offset + if reading.done: + break +``` + +`iter` takes `timeout`, `from_offset`, `idle_timeout` (1 s by default), `max_records` and +`max_bytes`. After the loop, `stop` tells which stop ended the read, and `done` is true when +the Stream is closed and the read gave its last Record. A new read from `last_offset` loses +and repeats no Record, also after `aclose` or a `break`. A read that the code leaves with +`break` and doesn't close runs on in Core until one of its stops. + +A read costs one marker in History for each Workflow Task that it spans, whatever the number +of Records. It has three other costs. + +- A read keeps its Workflow Task open. A Signal, an Update, a cancel or an Activity result + reaches the Workflow at the next Workflow Task, at the heartbeat of the task at the latest. + That is about 8 s with the default task timeout of 10 s. +- By default, `StreamReplaySource.STORE`, the marker holds a range of offsets, a count and a + digest, and no Record. A replay reads the Store again, so the Records must stay in the + Store for as long as the Workflow can replay. The option + `workflow_read_replay_from=StreamReplaySource.HISTORY` of `StreamConfig` puts the Records + into the marker, so a replay needs no Store. Then a read with no `max_bytes` gets a cap of + 256 KiB. +- A replay on a Worker with no cache reads the Store again on every Workflow Task. + +An append of Workflow code writes as the Workflow, with its Workflow Task as the epoch, and +records a marker with no body. A Store written in Python has no Rust runtime. On such a Store +an append runs in the Store's Local Activity, as before, and a read fails the Workflow. diff --git a/temporalio/exceptions.py b/temporalio/exceptions.py index 658012ee4..21a16835f 100644 --- a/temporalio/exceptions.py +++ b/temporalio/exceptions.py @@ -304,9 +304,10 @@ class StreamConfigError(RuntimeError): * This build of temporalio has no Stream runtime, or no Redis Stream Store. A Workflow's Stream call on a Worker whose client has no Stream config doesn't - raise this class. Workflow code gets an :py:class:`ApplicationError` with the - ``type`` ``StreamConfigError`` and ``non_retryable`` set, so an uncaught one fails - the Workflow. Workflow code can catch this error. A Workflow that catches it must + raise this class, and neither does a read of Workflow code on a Store written in + Python, which has no Rust runtime for Core to read through. Workflow code gets an + :py:class:`ApplicationError` with the ``type`` ``StreamConfigError`` and + ``non_retryable`` set, so an uncaught one fails the Workflow. Workflow code can catch this error. A Workflow that catches it must not go on to make the same call on a Worker that has a config, since the History then has no record of the first call. diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index e179facc7..d847e62d6 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -68,9 +68,15 @@ def __init__( ``stream_config`` takes the Stream config of the client that the worker had, so a replay makes the same commands. A history that records a - Stream call needs it. With no Stream config, Workflow code gets an error in - place of that call, and the replay fails with a nondeterminism error. A - replay calls no method of its Stores. + create, an append, a close or a describe needs it. With no Stream config, + Workflow code gets an error in place of that call, and the replay fails with a + nondeterminism error. A replay calls no method of its Stores from Python. + A read of Workflow code with ``iter`` that ran under the default + ``workflow_read_replay_from``, ``StreamReplaySource.STORE``, has no Records in + the history. Core reads them from the Store again, so the replay needs this + config and Stores that still have the Records. Without them the replay fails + its Workflow Task with a message that starts with ``StreamReplayError``. A read + that ran under the option ``HISTORY`` needs no Stream config. Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that diff --git a/tests/nexus/test_stream_store_workflow.py b/tests/nexus/test_stream_store_workflow.py index 53b68da38..c6633ebe0 100644 --- a/tests/nexus/test_stream_store_workflow.py +++ b/tests/nexus/test_stream_store_workflow.py @@ -651,8 +651,8 @@ async def test_workflow_read_fails_with_store(client: Client): task_queue=worker.task_queue, ) assert result == ( - "Workflow code can't read a Stream on a Store in this release. " - "Call an Activity that reads it through its client." + "Workflow code reads a Stream with `iter` of the Workflow Stream, not " + "with the ReadRecords operation." ) From bca1c8c6c9215897d7d94ba52b627ea3e786e8b5 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 04:06:27 -0700 Subject: [PATCH 10/59] Moved Core to the fixes of the Direct Stream calls. Core now orders a Workflow's close after its own appends and evicts a replay whose marker Records can't be read. Its protos didn't change, so the bridge protos and the visitor stay. --- temporalio/bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 92081b7cc..b0a3a52c7 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 92081b7ccb1596f2f6f9247094a49b94832fcff5 +Subproject commit b0a3a52c71e27aa68d06e69b35588d73369390f0 From 52d4d850300d41616be77616b10016ec3a81e4f7 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 04:06:28 -0700 Subject: [PATCH 11/59] Documented that a cap on the last Record of a closed Stream gives Done. Core ends such a read with Done, and counts the idle time only while a read waits for the Store. The stops, `iter_stream` and the README now say so. --- temporalio/contrib/streams/README.md | 4 +++- temporalio/nexus/system/_support.py | 17 +++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 06c2e8f93..ab1c74073 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -65,7 +65,9 @@ while True: `iter` takes `timeout`, `from_offset`, `idle_timeout` (1 s by default), `max_records` and `max_bytes`. After the loop, `stop` tells which stop ended the read, and `done` is true when -the Stream is closed and the read gave its last Record. A new read from `last_offset` loses +the Stream is closed and the read gave its last Record. That stop is `Done` also when the +last Record filled a cap. The idle time counts only while the read waits for the Store. A +new read from `last_offset` loses and repeats no Record, also after `aclose` or a `break`. A read that the code leaves with `break` and doesn't close runs on in Core until one of its stops. diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index 6f96358ff..7300c48f9 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -183,16 +183,20 @@ class StreamReadStop(enum.IntEnum): """ Done = 1 - """The Stream is closed, and the read gave its last Record.""" + """The Stream is closed, and the read gave its last Record. + + A read that gave the last Record of a closed Stream ends with this stop, also when + that Record filled ``max_records`` or ``max_bytes``. + """ MaxRecords = 2 - """The read gave ``max_records`` Records.""" + """The read gave ``max_records`` Records, and the Stream has more or is open.""" MaxBytes = 3 """The next Record would pass ``max_bytes``.""" Idle = 4 - """No Record came for ``idle_timeout``.""" + """No Record came for ``idle_timeout`` while the read waited for the Store.""" Timeout = 5 """``timeout`` passed since the read started.""" @@ -349,7 +353,7 @@ def iter_stream( as they arrive. The loop ends at the first stop. Then :py:attr:`StreamReading.stop` tells which stop it was, and :py:attr:`StreamReading.done` is true when the Stream is closed and the read gave its - last Record. A new read from :py:attr:`StreamReading.last_offset` loses and repeats + last Record. That stop is ``Done`` also when the last Record filled a cap. A new read from :py:attr:`StreamReading.last_offset` loses and repeats no Record, also after a close or a ``break``. .. code-block:: python @@ -396,8 +400,9 @@ def iter_stream( restart. ``None`` is no bound. from_offset: Read the Records after this offset. ``None`` is the start of the Stream. - idle_timeout: The read ends when no Record comes for this long. ``None`` is 1 s. - Zero gives what is in the Stream and doesn't wait. + idle_timeout: The read ends when no Record comes for this long while it waits for + the Store. Time in which its Records wait for the loop doesn't count. ``None`` + is 1 s. Zero gives what is in the Stream and doesn't wait. max_records: The most Records of the read. ``None`` is no cap. max_bytes: The most bytes of the Records of the read, each as its encoded ``StreamRecord``. The first Record always comes. ``None`` is no cap, unless From 4c4dda3156dc53f42f32a84ba9b889e7c1d4d00a Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 04:06:28 -0700 Subject: [PATCH 12/59] Tested failed replays, the capped Done and a run's end with a read open. A failed replay now reaches the caller with no cut-off, raised or in the result. New tests cover a second read after a `break`, a completion, a cancel and a continue-as-new. --- tests/nexus/test_stream_direct.py | 220 +++++++++++++++++++++++++++--- 1 file changed, 204 insertions(+), 16 deletions(-) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index ba7480a36..2bfd448a8 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -312,6 +312,23 @@ async def test_read_stops_at_max_records(client: Client): assert (rest.stop, rest.done) == ("Done", True) +async def test_read_that_takes_the_last_record_with_a_cap_is_done(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + # The cap takes the last Record of the closed Stream, so the stop is Done. + by_records = await _read_once(stream_client, ReadArgs(ref.id, max_records=3)) + assert by_records.bodies == ["a", "b", "c"] + assert (by_records.stop, by_records.done) == ("Done", True) + one = await _filled_stream(stream_client, ["a"]) + by_bytes = await _read_once(stream_client, ReadArgs(one.id, max_bytes=1)) + assert by_bytes.bodies == ["a"] + assert (by_bytes.stop, by_bytes.done) == ("Done", True) + # On an open Stream the same cap stops the read with its own stop. + open_ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + capped = await _read_once(stream_client, ReadArgs(open_ref.id, max_records=3)) + assert (capped.stop, capped.done) == ("MaxRecords", False) + + async def test_read_stops_at_max_bytes(client: Client): stream_client = _stream_client(client) ref = await _filled_stream(stream_client, ["a", "b", "c"]) @@ -507,29 +524,34 @@ async def test_replay_of_store_marker_reads_the_store(client: Client): ) -async def _failed_replay(history: WorkflowHistory, config: StreamConfig | None) -> str: - # Core fails the Workflow Task of the replay before any activation. The bound keeps a - # replay that never ends from holding the test. - with pytest.raises(Exception) as err: - await asyncio.wait_for( - Replayer( - workflows=[ReadOnceWorkflow], stream_config=config - ).replay_workflow(history), - timeout=10, - ) - return f"{type(err.value).__name__}: {err.value}" +async def _failed_replay( + history: WorkflowHistory, ref: StreamRef[str], config: StreamConfig | None +) -> str: + """Replay with and without a raise, and give the text of the failure.""" + replayer = Replayer(workflows=[ReadOnceWorkflow], stream_config=config) + with pytest.raises(RuntimeError) as err: + await replayer.replay_workflow(history) + result = await replayer.replay_workflow(history, raise_on_replay_failure=False) + assert isinstance(result.replay_failure, RuntimeError) + assert str(result.replay_failure) == str(err.value) + message = str(err.value) + assert f"StreamReplayError: Stream `{ref.id}` in Store `memory`" in message + return message async def test_replay_of_store_marker_with_no_stream_config_fails(client: Client): - history, _, _ = await _history_of_a_read(client, StreamReplaySource.STORE) - assert "StreamReplayError" in await _failed_replay(history, None) + history, ref, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + message = await _failed_replay(history, ref, None) + assert "this Worker has no Stream runtime for the Store `memory`" in message async def test_replay_of_store_marker_fails_when_records_are_gone(client: Client): - history, _, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + history, ref, _ = await _history_of_a_read(client, StreamReplaySource.STORE) # A new memory Store of the same name has none of the Records. - message = await _failed_replay(history, StreamConfig(stores=[MemoryStreamStore()])) - assert "StreamReplayError" in message + message = await _failed_replay( + history, ref, StreamConfig(stores=[MemoryStreamStore()]) + ) + assert "stream_not_found" in message async def test_replay_of_history_marker_needs_no_stream_config(client: Client): @@ -799,6 +821,172 @@ async def test_signal_reaches_a_running_read_and_closes_it(client: Client): assert result == [["a"], True, None] +# A second read after a `break`, and the ends of a run with a read open. + + +@activity.defn +async def append_and_close(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("x", "y") + await stream.close() + + +@workflow.defn +class BreakThenReadAgainWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[Any]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + first = stream.iter(idle_timeout=timedelta(seconds=30)) + got_first: list[str] = [] + async for record in first: + got_first.append(record.body) + if len(got_first) == 2: + break + # The first read runs on in Core while the second reads the same Stream. + second = stream.iter( + from_offset=first.last_offset, idle_timeout=timedelta(seconds=30) + ) + writer = workflow.start_activity( + append_and_close, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + got_second = [record.body async for record in second] + await writer + rest_of_first = [record.body async for record in first] + return [ + got_first, + got_second, + second.stop and second.stop.name, + rest_of_first, + first.stop and first.stop.name, + ] + + +async def test_second_read_after_a_break_gets_every_record(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + handle, result = await _run( + stream_client, BreakThenReadAgainWorkflow, ref.id, activities=[append_and_close] + ) + assert result == [ + ["a", "b"], + ["c", "x", "y"], + "Done", + ["c", "x", "y"], + "Done", + ] + await Replayer( + workflows=[BreakThenReadAgainWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +@workflow.defn +class CompleteWithOpenReadsWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + first = stream.iter(idle_timeout=timedelta(seconds=60)) + async for _ in first: + break + # A second read waits in `__anext__` in its own task while the Workflow returns. + second = stream.iter( + from_offset=first.last_offset, idle_timeout=timedelta(seconds=60) + ) + waiting = asyncio.create_task(second.__anext__()) + await asyncio.sleep(0) + assert not waiting.done() + return "completed" + + +async def test_workflow_completes_with_open_reads(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, CompleteWithOpenReadsWorkflow) as worker: + handle = await stream_client.start_workflow( + CompleteWithOpenReadsWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await asyncio.wait_for(handle.result(), timeout=20) == "completed" + await Replayer( + workflows=[CompleteWithOpenReadsWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +@workflow.defn +class CancelDuringReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[Any]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + reading = stream.iter(idle_timeout=timedelta(seconds=60)) + got: list[str] = [] + try: + async for record in reading: + got.append(record.body) + except asyncio.CancelledError: + await reading.aclose() + return [got, "cancelled", reading.stop] + return [got, "ended", reading.stop] + + +async def test_cancel_reaches_a_waiting_read(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, CancelDuringReadWorkflow) as worker: + handle = await stream_client.start_workflow( + CancelDuringReadWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + # The heartbeat of an open task comes at 80 % of its timeout. + task_timeout=timedelta(seconds=2), + ) + await asyncio.sleep(0.5) + await handle.cancel() + result = await asyncio.wait_for(handle.result(), timeout=30) + assert result == [["a"], "cancelled", None] + + +@workflow.defn +class ContinueWithOpenReadWorkflow: + @workflow.run + async def run(self, stream_id: str, from_offset: str | None) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + if from_offset is None: + first = stream.iter(idle_timeout=timedelta(seconds=60)) + async for _ in first: + break + assert first.last_offset is not None + workflow.continue_as_new(args=[stream_id, first.last_offset.hex()]) + reading = stream.iter( + from_offset=bytes.fromhex(from_offset), idle_timeout=timedelta() + ) + return [record.body async for record in reading] + + +async def test_continue_as_new_with_an_open_read(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + async with new_worker(stream_client, ContinueWithOpenReadWorkflow) as worker: + handle = await stream_client.start_workflow( + ContinueWithOpenReadWorkflow.run, + args=[ref.id, None], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + result = await asyncio.wait_for(handle.result(), timeout=20) + assert result == ["b", "c"] + first_run = stream_client.get_workflow_handle( + handle.id, run_id=handle.first_execution_run_id + ) + await Replayer( + workflows=[ContinueWithOpenReadWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await first_run.fetch_history()) + + # A Store written in Python. From 82caea0143e7a732723d327da61697c0677a34db Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 04:28:54 -0700 Subject: [PATCH 13/59] Replayed Direct calls whose ref names no Store with no Stream config. Core now gives a replayed call with no Store name the Store of its entry in History. So a Replayer with no Stream config replays a Direct append and a `HISTORY` read of such a ref. --- temporalio/bridge/sdk-core | 2 +- temporalio/worker/_replayer.py | 21 ++++--- temporalio/worker/_stream_store.py | 80 +++++++++++++++++-------- temporalio/worker/_workflow_instance.py | 13 +++- 4 files changed, 78 insertions(+), 38 deletions(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index b0a3a52c7..07b5a1109 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit b0a3a52c71e27aa68d06e69b35588d73369390f0 +Subproject commit 07b5a110984f46e63812bde6952f739d08274933 diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index d847e62d6..a7c55438d 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -68,15 +68,18 @@ def __init__( ``stream_config`` takes the Stream config of the client that the worker had, so a replay makes the same commands. A history that records a - create, an append, a close or a describe needs it. With no Stream config, - Workflow code gets an error in place of that call, and the replay fails with a - nondeterminism error. A replay calls no method of its Stores from Python. - A read of Workflow code with ``iter`` that ran under the default - ``workflow_read_replay_from``, ``StreamReplaySource.STORE``, has no Records in - the history. Core reads them from the Store again, so the replay needs this - config and Stores that still have the Records. Without them the replay fails - its Workflow Task with a message that starts with ``StreamReplayError``. A read - that ran under the option ``HISTORY`` needs no Stream config. + create, a close, a describe, or an append through a Store's Local Activity + needs it. With no Stream config, Workflow code gets an error in place of that + call, and the replay fails with a nondeterminism error. A replay calls no + method of its Stores from Python. An append that Core made in the Workflow + Task needs no Stream config, and neither does a read of Workflow code with + ``iter`` that ran under the option ``HISTORY`` of ``workflow_read_replay_from``. + That holds also when the ref names no Store, since Core takes the Store from the + history. A read that ran under the default, ``StreamReplaySource.STORE``, has no + Records in the history. Core reads them from the Store again, so the replay + needs this config and Stores that still have the Records. Without them the + replay fails with a :py:class:`RuntimeError` whose message has + ``StreamReplayError``, the Stream and the cause. Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index c63d52945..87407705f 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -86,6 +86,59 @@ def _stream_error_from_core( # pyright: ignore[reportUnusedFunction] return None +def _start_direct_append( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: temporalio.api.streamservice.v1.AppendRecordsRequest, + store_name: str, +) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Send an append of a Workflow Producer to Core, which makes it in the Workflow Task.""" + payload_converter, _ = instance._system_nexus_converters(input) + append = temporalio.bridge.proto.workflow_commands.AppendToStream( + stream=temporalio.api.stream.v1.StreamRef(id=request.ref.id, store=store_name), + bodies=[record.body for record in request.records], + producer=request.producer, + sequence=request.sequence, + ) + append.producer.workflow.ClearField("epoch") + appended = instance._start_stream_append(append) + + async def result() -> OutputT: + response = await appended + return payload_converter._from_transfer_value( + temporalio.api.streamservice.v1.AppendRecordsResponse( + first_offset=response.first_offset, + last_offset=response.last_offset, + ), + input.output_type, + ) + + return _StreamStoreOperationHandle( + instance.create_task(result(), name=f"stream store: {input.operation_name}") + ) + + +def _start_replayed_direct_append( # pyright: ignore[reportUnusedFunction] + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], +) -> temporalio.workflow.NexusOperationHandle[OutputT] | None: + """Replay a Direct append on a Worker with no Stream config, or give ``None``. + + The marker of a Direct append holds its result, so its replay needs no Store. A ref + that names no Store sends an empty name, and Core takes the Store of History. + """ + if input.operation_name != "AppendRecords": + return None + payload_converter, _ = instance._system_nexus_converters(input) + request = payload_converter._to_transfer_value(input.input) + if ( + not isinstance(request, temporalio.api.streamservice.v1.AppendRecordsRequest) + or request.producer.WhichOneof("kind") != "workflow" + ): + return None + return _start_direct_append(instance, input, request, request.ref.store) + + def _read_command( # pyright: ignore[reportUnusedFunction] reading: temporalio.nexus.system._support.StreamReading[Any], store_name: str, @@ -193,32 +246,7 @@ def start_operation( ): # Core appends inside the Workflow Task and sets the epoch of the Producer # to that task. A Producer of another kind keeps the Local Activity path. - append = temporalio.bridge.proto.workflow_commands.AppendToStream( - stream=temporalio.api.stream.v1.StreamRef( - id=request.ref.id, store=store.name - ), - bodies=[record.body for record in request.records], - producer=request.producer, - sequence=request.sequence, - ) - append.producer.workflow.ClearField("epoch") - appended = instance._start_stream_append(append) - - async def direct_result() -> OutputT: - response = await appended - return payload_converter._from_transfer_value( - temporalio.api.streamservice.v1.AppendRecordsResponse( - first_offset=response.first_offset, - last_offset=response.last_offset, - ), - input.output_type, - ) - - return _StreamStoreOperationHandle( - instance.create_task( - direct_result(), name=f"stream store: {input.operation_name}" - ) - ) + return _start_direct_append(instance, input, request, store.name) tails = instance._stream_write_tails tail_key = (store.name, ref.id) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index a1ce88901..8663ec9de 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -101,6 +101,7 @@ from ._stream_store import ( _no_stream_config_error, _read_command, + _start_replayed_direct_append, _stream_error_from_core, _WorkflowStreamStore, ) @@ -1859,7 +1860,8 @@ def workflow_start_stream_read( start = store.read_command(reading) elif self._is_replaying: # A replay of a read whose marker holds its Records reads no Store, so it needs - # no Stream config. The ref names the Store, as the first run had it. + # no Stream config. A ref that names no Store sends an empty name, and Core + # takes the Store of History. start = _read_command( reading, reading._stream.ref.store or "", records_in_history=True ) @@ -2497,7 +2499,14 @@ async def _schedule_system_nexus_operation( if input.service == _STREAM_SERVICE.name: store = self._stream_store if store is None: - raise _no_stream_config_error() + replayed = ( + _start_replayed_direct_append(self, input) + if self._is_replaying + else None + ) + if replayed is None: + raise _no_stream_config_error() + return replayed return store.start_operation(self, input) return await self._outbound_start_nexus_operation(input) From 89d4cdb5852ec3b5d2dc1701a2569cebaa5ec42e Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 04:29:04 -0700 Subject: [PATCH 14/59] Woke every task that waits on a read, and kept a bad body readable. A second task that waited on one read replaced the future of the first, which then never woke. A Record that failed to decode moved `last_offset` past it, so a new read lost it. --- temporalio/nexus/system/_support.py | 47 ++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index 7300c48f9..f25285322 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -34,6 +34,11 @@ _BodyT = typing.TypeVar("_BodyT") +if typing.TYPE_CHECKING: + _RecordDecoder: typing.TypeAlias = collections.abc.Callable[ + [temporalio.api.stream.v1.StreamRecord], StreamRecord[_BodyT] + ] + class SignalWithStartWorkflowModelRequest(typing.Protocol): namespace: str @@ -255,7 +260,9 @@ def __init__(self, stream: typing.Any, options: _StreamReadOptions) -> None: self._end: tuple[StreamReadStop | None, bool, BaseException | None] | None = ( None ) - self._waiter: asyncio.Future[None] | None = None + # Each task that waits in `__anext__` has its own future, and a page, an end and a + # close wake every one of them. + self._waiters: list[asyncio.Future[None]] = [] self._finished = False def __aiter__(self) -> typing_extensions.Self: @@ -275,20 +282,24 @@ async def __anext__(self) -> "StreamRecord[_BodyT]": raise StopAsyncIteration if self._seq is None: temporalio.workflow._Runtime.current().workflow_start_stream_read(self) - while not self._records and self._end is None: - self._waiter = asyncio.get_running_loop().create_future() + while not self._records and self._end is None and not self._finished: + waiter: asyncio.Future[None] = asyncio.get_running_loop().create_future() + self._waiters.append(waiter) try: - await self._waiter + await waiter finally: - self._waiter = None - if self._finished: - raise StopAsyncIteration + self._waiters.remove(waiter) + if self._finished: + raise StopAsyncIteration if self._records: record = self._records.popleft() if self._decode is None: raise RuntimeError("The Stream read has no decoder") + # A Record that fails to decode raises here, and `last_offset` stays before it, + # so a new read from `last_offset` gets it again. + decoded = self._decode(record) self.last_offset = record.offset - return self._decode(record) + return decoded assert self._end is not None stop, done, error = self._end self._finished = True @@ -313,13 +324,17 @@ async def aclose(self) -> None: ) self._finished = True self._records.clear() - if self._waiter is not None and not self._waiter.done(): - self._waiter.set_result(None) + self._wake() + + def _wake(self) -> None: + for waiter in self._waiters: + if not waiter.done(): + waiter.set_result(None) def _started( self, seq: int, - decode: "collections.abc.Callable[[temporalio.api.stream.v1.StreamRecord], StreamRecord[_BodyT]]", + decode: "_RecordDecoder[_BodyT]", ) -> None: self._seq = seq self._decode = decode @@ -332,8 +347,7 @@ def _take( self._records.extend(records) if end is not None: self._end = end - if self._waiter is not None and not self._waiter.done(): - self._waiter.set_result(None) + self._wake() def iter_stream( @@ -353,8 +367,11 @@ def iter_stream( as they arrive. The loop ends at the first stop. Then :py:attr:`StreamReading.stop` tells which stop it was, and :py:attr:`StreamReading.done` is true when the Stream is closed and the read gave its - last Record. That stop is ``Done`` also when the last Record filled a cap. A new read from :py:attr:`StreamReading.last_offset` loses and repeats - no Record, also after a close or a ``break``. + last Record. That stop is ``Done`` also when the last Record filled a cap. A new + read from :py:attr:`StreamReading.last_offset` loses and repeats no Record, also + after a close, a ``break``, or a Record whose body fails to decode. Such a Record + raises from its step of the loop, and ``last_offset`` stays before it. Two tasks can + step one read. Each Record goes to one of them, and both end when the read ends. .. code-block:: python From 02ad6a316aefc2f4e113d23286c18ed8232105ba Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 04:29:04 -0700 Subject: [PATCH 15/59] Ran the codec on the Stream bodies of Direct calls with no context. The Local Activity path, a client and an Activity run it with no serialization context. A codec with a context then wrote bodies that no other reader of the Stream could decode. --- temporalio/worker/_command_aware_visitor.py | 25 ++++++++++++++++----- temporalio/worker/_workflow_instance.py | 5 +++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index e6180c1b5..282b2d547 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -17,6 +17,7 @@ ResolveRequestCancelExternalWorkflow, ResolveSignalExternalWorkflow, ResolveStreamAppend, + ResolveStreamRead, StreamReadEnd, ) from temporalio.bridge.proto.workflow_commands.workflow_commands_pb2 import ( @@ -40,6 +41,8 @@ class CommandInfo: command_type: CommandType.ValueType command_seq: int update_protocol_instance_id: str | None = None + stream: bool = False + """Whether the payloads are the bodies of a Stream append or the Records of a read.""" current_command_info: contextvars.ContextVar[CommandInfo | None] = ( @@ -79,16 +82,26 @@ def __init__( ) self.skip_stream_bodies = skip_stream_bodies - # A Store keeps the bodies of a Workflow's append as the Local Activity path writes - # them, with the codec and with no external storage, so other readers can decode them. - # Each body is one Record, so the codec gets each body alone, as for that path. + # A Store keeps the bodies of a Workflow's append as the Local Activity path, a client + # and an Activity write them: with the codec, with no serialization context and with no + # external storage. So every reader decodes them. Each body is one Record, so the codec + # gets each body alone, as for those paths. async def _visit_coresdk_workflow_commands_AppendToStream( self, fs: VisitorFunctions, o: AppendToStream ) -> None: if self.skip_stream_bodies: return - for body in o.bodies: - await self._visit_temporal_api_common_v1_Payload(fs, body) + with current_command(CommandType.COMMAND_TYPE_UNSPECIFIED, o.seq, stream=True): + for body in o.bodies: + await self._visit_temporal_api_common_v1_Payload(fs, body) + + # The Records of a read come from any writer of the Stream, so the codec decodes them + # with no serialization context, as a client and an Activity do. + async def _visit_coresdk_workflow_activation_ResolveStreamRead( + self, fs: VisitorFunctions, o: ResolveStreamRead + ) -> None: + with current_command(CommandType.COMMAND_TYPE_UNSPECIFIED, o.seq, stream=True): + await super()._visit_coresdk_workflow_activation_ResolveStreamRead(fs, o) # Core builds the failure of a Stream read or append itself, with no codec, so no pass # may decode it. @@ -248,6 +261,7 @@ def current_command( command_seq: int, *, update_protocol_instance_id: str | None = None, + stream: bool = False, ) -> Iterator[None]: """Context manager for setting command info.""" token = current_command_info.set( @@ -255,6 +269,7 @@ def current_command( command_type=command_type, command_seq=command_seq, update_protocol_instance_id=update_protocol_instance_id, + stream=stream, ) ) try: diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 8663ec9de..fd9eb5e75 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -2702,6 +2702,11 @@ def get_serialization_context( workflow_id=self._info.workflow_id, ) + if command_info.stream: + # Stream bodies take the codec with no context on every path, so a client, an + # Activity and a Workflow read what each other wrote. + return None + if self._nexus_serialization_context and command_info.command_type in ( temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION, From 2068b11145cead8e90337daa52da3a0386b85438 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 04:28:54 -0700 Subject: [PATCH 16/59] Gave Core the Store runtimes only from a Worker with Workflows. Only Workflow code makes Direct calls, so an Activity-only Worker with a bad Redis option starts again. The Store docstrings now say when a Worker builds the runtime. --- temporalio/client/_stream_store.py | 4 ++-- temporalio/contrib/redis/streams.py | 14 +++++++++----- temporalio/testing/_stream_store.py | 4 ++-- temporalio/worker/_worker.py | 6 +++++- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index 6687edb0f..b35c1c57c 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -224,8 +224,8 @@ async def call_from_workflow( class _BridgeStreamStore(StreamStore): # pyright: ignore[reportUnusedClass] """A Store that runs on the Stream runtime of the bridge. - It builds the runtime at its first call, so the Store can be built before the - Runtime and its Tokio threads exist. + It builds the runtime at its first call, or when a Worker with Workflows gives it to + Core, so the Store can be built before the Runtime and its Tokio threads exist. """ def __init__( diff --git a/temporalio/contrib/redis/streams.py b/temporalio/contrib/redis/streams.py index 8a366fe5f..ab8136d84 100644 --- a/temporalio/contrib/redis/streams.py +++ b/temporalio/contrib/redis/streams.py @@ -31,7 +31,9 @@ class RedisStreamStore(temporalio.client._stream_store._BridgeStreamStore): It runs the Stream runtime of the bridge on the Redis Store of the ``temporalio-streams-redis`` crate. It connects at its first call, so building it needs no server, and a server that is down fails that call. A failed call - raises :py:class:`temporalio.exceptions.StreamError`. + raises :py:class:`temporalio.exceptions.StreamError`. A Worker with Workflows gives + Core this runtime when the Worker is built, so Core can read and append for Workflow + code. A key has no namespace, so a Stream id is unique in one Redis and key prefix. Give each namespace its own key prefix to keep their Streams apart. @@ -54,10 +56,12 @@ def __init__( name: str = "redis", runtime: temporalio.runtime.Runtime | None = None, ) -> None: - """Create a Redis Store. It builds its Stream runtime and connects at its first call. + """Create a Redis Store. It builds its Stream runtime at its first use. - A bad option raises ``ValueError`` at that first call, with the message of the - Redis crate. + The first use is the first call, or the build of a Worker with Workflows on a + client with this Store, which gives Core the runtime. A bad option raises + ``ValueError`` there, with the message of the Redis crate. The Store connects at + its first call. Args: url: A ``redis://`` or ``rediss://`` URL, or the seed URLs of a cluster. @@ -79,7 +83,7 @@ def __init__( name: Name of the Store, which the refs of its Streams carry, so that every process can route a ref that it gets. runtime: Runtime whose Tokio threads run the Store. ``None`` takes - :py:meth:`temporalio.runtime.Runtime.default` at the first call. + :py:meth:`temporalio.runtime.Runtime.default` at the first use. """ temporalio.client._stream_store._check_not_negative( retention=retention, diff --git a/temporalio/testing/_stream_store.py b/temporalio/testing/_stream_store.py index 1220bbc5b..42c8500fd 100644 --- a/temporalio/testing/_stream_store.py +++ b/temporalio/testing/_stream_store.py @@ -28,7 +28,7 @@ def __init__( producer_idle_ttl: timedelta | None = None, runtime: temporalio.runtime.Runtime | None = None, ) -> None: - """Create a memory Store. It builds its Stream runtime at its first call. + """Create a memory Store. It builds its Stream runtime at its first use. Args: name: Name of the Store, which the refs of its Streams carry. @@ -37,7 +37,7 @@ def __init__( producer_idle_ttl: How long an open Stream stays after the last write of any Producer. ``None`` keeps an open Stream until its close. runtime: Runtime whose Tokio threads run the Store. ``None`` takes - :py:meth:`temporalio.runtime.Runtime.default` at the first call. + :py:meth:`temporalio.runtime.Runtime.default` at the first use. """ temporalio.client._stream_store._check_not_negative( retention=retention, producer_idle_ttl=producer_idle_ttl diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index 7c870853e..8d7b7ee55 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -724,7 +724,11 @@ def check_activity(activity: str): disable_payload_error_limit=config.get( "disable_payload_error_limit", False ), - stream_runtimes=_bridge_stream_runtimes(stream_config), + # Only Workflow code makes Direct calls, so a Worker with no Workflows + # builds no runtime, as it builds no Store Activity. + stream_runtimes=_bridge_stream_runtimes( + stream_config if self._workflow_worker is not None else None + ), ), ) From 47229877ebc5947198b3179ac56f65860be4ad0a Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 04:28:54 -0700 Subject: [PATCH 17/59] Tested the fixes of the review, and wrapped three long lines. Each new test failed before its fix: a shared read, the codec context, replays of an unnamed ref, a bad body, and an Activity-only Worker with a bad Redis option. --- .github/workflows/ci.yml | 2 +- temporalio/exceptions.py | 6 +- tests/nexus/test_stream_direct.py | 261 +++++++++++++++++++++++++++++- 3 files changed, 262 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91342ce5a..ba53ed32b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -228,7 +228,7 @@ jobs: - run: uv tool install poethepoet - run: uv sync --all-extras - run: cargo install --locked nexgen --version 0.2.8 --features advanced --force - # The Stream package needs `@nexus.helper` with `sync=true`, which only a fork of nexgen has yet. + # The Stream package needs `@nexus.helper` with `sync=true`, which only a nexgen fork has. - run: cargo install --locked --git https://github.com/moetemp/nexgen --rev 42c7e43799f97c837346f1562db13493d9869858 --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen - run: echo "NEX_GEN_STREAM_BIN=$RUNNER_TEMP/nexgen-stream/bin/nexgen" >> "$GITHUB_ENV" - run: poe build-develop diff --git a/temporalio/exceptions.py b/temporalio/exceptions.py index 21a16835f..526c2eb47 100644 --- a/temporalio/exceptions.py +++ b/temporalio/exceptions.py @@ -307,9 +307,9 @@ class StreamConfigError(RuntimeError): raise this class, and neither does a read of Workflow code on a Store written in Python, which has no Rust runtime for Core to read through. Workflow code gets an :py:class:`ApplicationError` with the ``type`` ``StreamConfigError`` and - ``non_retryable`` set, so an uncaught one fails the Workflow. Workflow code can catch this error. A Workflow that catches it must - not go on to make the same call on a Worker that has a config, since the History - then has no record of the first call. + ``non_retryable`` set, so an uncaught one fails the Workflow. Workflow code can + catch this error. A Workflow that catches it must not go on to make the same call on + a Worker that has a config, since the History then has no record of the first call. .. warning:: This API is experimental and subject to change. diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 2bfd448a8..c07b5c99b 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -4,7 +4,7 @@ import asyncio import uuid -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Sequence from dataclasses import dataclass from datetime import timedelta from typing import Any @@ -12,6 +12,7 @@ import pytest from temporalio import activity, workflow +from temporalio.api.common.v1 import Payload from temporalio.api.enums.v1 import EventType, StreamErrorReason from temporalio.client import ( Client, @@ -22,7 +23,13 @@ WorkflowHistory, ) from temporalio.common import RawValue -from temporalio.converter import DataConverter +from temporalio.converter import ( + DataConverter, + PayloadCodec, + SerializationContext, + WithSerializationContext, + WorkflowSerializationContext, +) from temporalio.exceptions import ApplicationError, StreamError from temporalio.nexus.system.stream_service import ( Producer, @@ -31,7 +38,7 @@ StreamRef, ) from temporalio.testing import MemoryStreamStore -from temporalio.worker import Replayer +from temporalio.worker import Replayer, Worker from tests.helpers import new_worker from tests.helpers.streams import DictStreamStore from tests.test_converter import SimpleCodec @@ -1134,3 +1141,251 @@ async def test_measure_read_of_200_records_over_2_seconds( f"\n{replay_from.name}: {_completed_tasks(history)} Workflow Tasks, " f"{len(sizes)} markers, {sum(sizes)} marker bytes, sizes {sizes}" ) + + +# Two tasks that share one read. + + +@workflow.defn +class SharedReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[Any]: + reading = workflow.get_stream(ref, body_type=str).iter() + got: list[str] = [] + + async def drain() -> int: + count = 0 + async for record in reading: + got.append(record.body) + count += 1 + return count + + # Both tasks wait in `__anext__` before the first page comes. + first = asyncio.create_task(drain()) + second = asyncio.create_task(drain()) + try: + await workflow.wait_condition( + lambda: first.done() and second.done(), timeout=timedelta(seconds=5) + ) + except asyncio.TimeoutError: + first.cancel() + second.cancel() + return ["stuck", got] + assert reading.stop is not None + return [sorted(got), first.result() + second.result(), reading.stop.name] + + +async def test_two_tasks_share_one_read(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + _, result = await _run(stream_client, SharedReadWorkflow, ref) + # Each Record goes to one of the tasks, and both end when the read ends. + assert result == [["a", "b", "c"], 3, "Done"] + + +# A codec that uses its serialization context. + + +@dataclass(frozen=True) +class ContextCodec(PayloadCodec, WithSerializationContext): + """Writes the Workflow id of its context into each payload, and decodes only its own.""" + + workflow_id: str = "" + + def with_context(self, context: SerializationContext) -> ContextCodec: + if isinstance(context, WorkflowSerializationContext): + return ContextCodec(context.workflow_id) + return ContextCodec("") + + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + return [ + Payload( + metadata={"encoding": b"binary/ctx", "ctx": self.workflow_id.encode()}, + data=payload.SerializeToString(), + ) + for payload in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + out = [] + for payload in payloads: + if payload.metadata.get("encoding") != b"binary/ctx": + out.append(payload) + continue + if payload.metadata["ctx"] != self.workflow_id.encode(): + raise ApplicationError( + f"Encoded for {payload.metadata['ctx']!r}, not " + f"{self.workflow_id!r}", + non_retryable=True, + ) + out.append(Payload.FromString(payload.data)) + return out + + +@workflow.defn +class AppendThenReadAllWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + await stream.append("from-workflow") + return [record.body async for record in stream.iter(idle_timeout=timedelta())] + + +@pytest.mark.parametrize("store_kind", ["memory", "dict"]) +async def test_codec_runs_with_no_context_on_stream_bodies( + client: Client, store_kind: str +): + store = MemoryStreamStore() if store_kind == "memory" else DictStreamStore() + config = client.config() + config["stream_config"] = StreamConfig(stores=[store]) + config["data_converter"] = DataConverter(payload_codec=ContextCodec()) + stream_client = Client(**config) + ref = await _filled_stream(stream_client, ["from-client"], close=False) + if store_kind == "dict": + # A Store written in Python has no Direct read, so only the append runs. + await _run(stream_client, AppendOneWorkflow, ref) + else: + _, bodies = await _run(stream_client, AppendThenReadAllWorkflow, ref) + assert bodies == ["from-client", "from-workflow"] + # The client decodes with no context, so it reads the Workflow's body too. + assert [record.body for record in await _read_all(stream_client, ref)] == [ + "from-client", + "from-workflow", + ] + raw_config = client.config() + raw_config["stream_config"] = StreamConfig(stores=[store]) + raw = Client(**raw_config).get_stream(StreamRef(id=ref.id), body_type=RawValue) + stored = [record.body.payload async for record in raw.iter_read_items()] + assert [body.metadata["ctx"] for body in stored] == [b"", b""] + + +@workflow.defn +class AppendOneWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + await workflow.get_stream(ref, body_type=str).append("from-workflow") + + +# Replays with no Stream config of calls whose ref names no Store. + + +@workflow.defn +class UnnamedRefReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + return [record.body async for record in stream.iter()] + + +async def _unnamed_read_history( + client: Client, replay_from: StreamReplaySource +) -> tuple[WorkflowHistory, str]: + stream_client = _stream_client(client, replay_from=replay_from) + ref = await _filled_stream(stream_client, ["a", "b"]) + handle, result = await _run(stream_client, UnnamedRefReadWorkflow, ref.id) + assert result == ["a", "b"] + return await handle.fetch_history(), ref.id + + +async def test_history_replay_of_an_unnamed_ref_needs_no_config(client: Client): + history, _ = await _unnamed_read_history(client, StreamReplaySource.HISTORY) + await Replayer(workflows=[UnnamedRefReadWorkflow]).replay_workflow(history) + + +async def test_store_replay_of_an_unnamed_ref_with_no_config_fails(client: Client): + history, stream_id = await _unnamed_read_history(client, StreamReplaySource.STORE) + with pytest.raises(RuntimeError) as err: + await Replayer(workflows=[UnnamedRefReadWorkflow]).replay_workflow(history) + message = str(err.value) + assert f"StreamReplayError: Stream `{stream_id}` in Store `memory`" in message + assert "this Worker has no Stream runtime for the Store `memory`" in message + assert "Nondeterminism" not in message + + +@workflow.defn +class UnnamedRefAppendWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str | None: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + appended = await stream.append("a") + return _hex(appended.last_offset) + + +async def test_replay_of_an_unnamed_append_needs_no_config(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run(stream_client, UnnamedRefAppendWorkflow, ref.id) + history = await handle.fetch_history() + assert _markers(history) == ["core_stream"] + await Replayer(workflows=[UnnamedRefAppendWorkflow]).replay_workflow(history) + + +# A Record whose body fails to decode. + + +@dataclass +class Point: + x: int + + +@workflow.defn +class BadBodyWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[Any]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=Point) + reading = stream.iter() + events: list[Any] = [] + while True: + try: + record = await reading.__anext__() + events.append(["got", record.body.x]) + except StopAsyncIteration: + break + except Exception as err: + events.append(["error", type(err).__name__]) + break + # A new read from `last_offset` gets the Record that failed again. + again = stream.iter(from_offset=reading.last_offset) + try: + await again.__anext__() + events.append(["again", "decoded"]) + except Exception as err: + events.append(["again", type(err).__name__]) + await again.aclose() + return events + + +async def test_last_offset_stays_before_a_body_that_fails_to_decode(client: Client): + stream_client = _stream_client(client) + stream = await stream_client.create_stream(id=f"s-{uuid.uuid4()}", body_type=object) + await stream.append({"x": 1}, "not a point", {"x": 3}) + await stream.close() + _, events = await _run(stream_client, BadBodyWorkflow, stream.ref.id) + assert events == [["got", 1], ["error", "TypeError"], ["again", "TypeError"]] + + +# A Worker with no Workflows and a Store with a bad option. + + +@activity.defn +async def noop() -> None: + pass + + +async def test_worker_without_workflows_builds_no_store_runtime(client: Client): + from temporalio.contrib.redis.streams import RedisStreamStore + + config = client.config() + config["stream_config"] = StreamConfig(stores=[RedisStreamStore("not a url")]) + # Such a Worker starts, as it did before Direct calls, since it gives Core no runtime. + async with Worker( + Client(**config), task_queue=f"tq-{uuid.uuid4()}", activities=[noop] + ): + pass + # A Worker with Workflows gives Core the runtime, so it builds it at once. + with pytest.raises(ValueError, match="The Redis URL is wrong"): + Worker( + Client(**config), + task_queue=f"tq-{uuid.uuid4()}", + workflows=[AppendOneWorkflow], + ) From 82487dd41f74d72d509bdb97096fdda684d5e44a Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 04:30:10 -0700 Subject: [PATCH 18/59] Moved Core to its fix of the transition coverage report. The commit changes only a module that Core builds for its tests, so the bridge needs no rebuild. --- temporalio/bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 07b5a1109..0754feced 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 07b5a110984f46e63812bde6952f739d08274933 +Subproject commit 0754feced59bce59b1b3fa85258bb306c3f18abe From 2b4cf5474881234f9a72e1bbabc6d294f943c59c Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 04:30:53 -0700 Subject: [PATCH 19/59] Fitted the Redis tests to the Direct append and tested iter on Redis. A Workflow's append on Redis now writes as the Workflow, with its Workflow Task as the epoch. A new test reads a running Activity's Stream with `iter`, then replays it from Redis. --- tests/contrib/redis/test_streams.py | 69 ++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/tests/contrib/redis/test_streams.py b/tests/contrib/redis/test_streams.py index b60a7e4df..ed05b432d 100644 --- a/tests/contrib/redis/test_streams.py +++ b/tests/contrib/redis/test_streams.py @@ -26,6 +26,7 @@ ) from temporalio.nexus.system.stream_service.client import Stream from temporalio.nexus.system.stream_service.workflow import create_stream +from temporalio.worker import Replayer from tests.helpers import new_worker _REDIS_URL = os.environ.get("STREAMS_REDIS_URL") @@ -172,13 +173,79 @@ async def test_workflow_appends_and_closes(client: Client): assert record.producer.id == f"wf:{run_id}" assert isinstance(record.producer.kind, ProducerKindWorkflow) assert record.producer.kind.value.run_id == run_id - assert record.producer.kind.value.epoch is not None + epoch = record.producer.kind.value.epoch + assert epoch is not None + assert epoch.started_event_id > 0 + assert epoch.attempt == 1 described = await stream.describe() assert described.status == StreamStatus.StreamStatusClosed err = await _stream_error(stream.append("c")) assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_CLOSED +@activity.defn +async def append_in_two_steps(stream_id: str, seen_id: str) -> None: + client = activity.client() + stream = client.get_stream(StreamRef(id=stream_id), body_type=str) + seen = client.get_stream(StreamRef(id=seen_id), body_type=str) + await stream.append("a", "b") + # The second step waits until the Workflow says that it has the first, so the + # Workflow reads while the Activity runs. + async for _ in seen.iter_read_items(wait=timedelta(seconds=30)): + break + await stream.append("c", "d") + await stream.close() + + +@workflow.defn +class IterWhileActivityRunsWorkflow: + @workflow.run + async def run(self, stream_id: str, seen_id: str) -> list[str]: + writer = workflow.start_activity( + append_in_two_steps, + args=[stream_id, seen_id], + start_to_close_timeout=timedelta(seconds=60), + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + seen = workflow.get_stream(StreamRef(id=seen_id), body_type=str) + got: list[str] = [] + last = None + while True: + reading = stream.iter(from_offset=last) + async for record in reading: + got.append(record.body) + if record.body == "b": + await seen.append("the first step") + last = reading.last_offset + if reading.done: + assert reading.stop is not None + got.append(reading.stop.name) + break + await writer + return got + + +async def test_workflow_iter_reads_an_activity_on_redis(client: Client): + stream_client = _with_store(client, _store()) + stream = await _create_stream(stream_client) + seen = await _create_stream(stream_client) + async with new_worker( + stream_client, IterWhileActivityRunsWorkflow, activities=[append_in_two_steps] + ) as worker: + handle = await stream_client.start_workflow( + IterWhileActivityRunsWorkflow.run, + args=[stream.ref.id, seen.ref.id], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await handle.result() == ["a", "b", "c", "d", "Done"] + # The default replay source keeps no Record in History, so the replay reads Redis. + await Replayer( + workflows=[IterWhileActivityRunsWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + async def test_activity_appends_with_activity_producer(client: Client): stream_client = _with_store(client, _store()) stream = await _create_stream(stream_client) From 8b48c58b604cfb87d48d705de8f78a79791d8aa8 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 10:18:01 -0700 Subject: [PATCH 20/59] Raised a failed read's error at every step of its loop. Two tasks can share one read. The task that didn't take the failure got a clean end, so it could miss that the read failed. --- temporalio/nexus/system/_support.py | 21 +++++++++++++++------ tests/nexus/test_stream_direct.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index f25285322..3e82b67b7 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -264,6 +264,7 @@ def __init__(self, stream: typing.Any, options: _StreamReadOptions) -> None: # close wake every one of them. self._waiters: list[asyncio.Future[None]] = [] self._finished = False + self._error: BaseException | None = None def __aiter__(self) -> typing_extensions.Self: """Give this read, since it is its own iterator.""" @@ -274,12 +275,13 @@ async def __anext__(self) -> "StreamRecord[_BodyT]": Raises: StopAsyncIteration: The read ended at a stop, or the code closed it. - temporalio.exceptions.StreamError: The read failed. + temporalio.exceptions.StreamError: The read failed. Every later step raises + it again. """ import temporalio.workflow if self._finished: - raise StopAsyncIteration + self._raise_end() if self._seq is None: temporalio.workflow._Runtime.current().workflow_start_stream_read(self) while not self._records and self._end is None and not self._finished: @@ -290,7 +292,7 @@ async def __anext__(self) -> "StreamRecord[_BodyT]": finally: self._waiters.remove(waiter) if self._finished: - raise StopAsyncIteration + self._raise_end() if self._records: record = self._records.popleft() if self._decode is None: @@ -303,10 +305,16 @@ async def __anext__(self) -> "StreamRecord[_BodyT]": assert self._end is not None stop, done, error = self._end self._finished = True - if error is not None: - raise error + self._error = error self.stop = stop self.done = done + self._raise_end() + + def _raise_end(self) -> typing.NoReturn: + # A second task that shares the read must not take a failure for a clean end, so + # every step after a failure raises it. + if self._error is not None: + raise self._error raise StopAsyncIteration async def aclose(self) -> None: @@ -371,7 +379,8 @@ def iter_stream( read from :py:attr:`StreamReading.last_offset` loses and repeats no Record, also after a close, a ``break``, or a Record whose body fails to decode. Such a Record raises from its step of the loop, and ``last_offset`` stays before it. Two tasks can - step one read. Each Record goes to one of them, and both end when the read ends. + step one read. Each Record goes to one of them, and both end when the read ends. A + read that failed raises its error at every step, in each task. .. code-block:: python diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index c07b5c99b..6ad7689f7 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -1183,6 +1183,35 @@ async def test_two_tasks_share_one_read(client: Client): assert result == [["a", "b", "c"], 3, "Done"] +@workflow.defn +class SharedFailedReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + reading = workflow.get_stream(ref, body_type=str).iter() + + async def step() -> str: + try: + await reading.__anext__() + except StreamError as err: + return StreamErrorReason.Name(err.reason) + except StopAsyncIteration: + return "end" + return "record" + + # Both tasks wait in `__anext__` before the failure comes. + first = asyncio.create_task(step()) + second = asyncio.create_task(step()) + return [await first, await second, await step()] + + +async def test_two_tasks_share_a_failed_read(client: Client): + stream_client = _stream_client(client) + ref: StreamRef[str] = StreamRef(id=f"stream-{uuid.uuid4()}") + _, result = await _run(stream_client, SharedFailedReadWorkflow, ref) + # No task takes the failure for a clean end, and a later step raises it again. + assert result == ["STREAM_ERROR_REASON_NOT_FOUND"] * 3 + + # A codec that uses its serialization context. From bdc5fb24d6bf95fd4cd7fe64e6560a3fe38e93ad Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 10:18:01 -0700 Subject: [PATCH 21/59] Said that a Store isn't meant to be written in Python. Such a Store has no Rust runtime, so Workflow code loses the Direct calls on it. The docs say what it loses. --- temporalio/client/_stream_store.py | 11 +++++++++-- temporalio/contrib/streams/README.md | 5 +++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index b35c1c57c..c14dbb41a 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -105,8 +105,15 @@ class StreamStore(ABC): :py:class:`temporalio.contrib.redis.streams.RedisStreamStore`, or :py:class:`temporalio.testing.MemoryStreamStore` in tests. - To keep Streams in another system, write a Store that implements :py:attr:`name` - and the five operations. Each operation takes and gives a message of + The SDK's Stores run on the bridge, in Rust, and this class isn't meant to be + subclassed in Python. A Store that is written in Python works for a client and an + Activity, for example as a double in a test. Workflow code loses the Direct calls on + it. An append of Workflow code then runs in a Local Activity, with one marker in + History for each call, and ``iter`` fails the Workflow, so Workflow code can't read + its Streams. + + Such a Store implements :py:attr:`name` and the five operations. Each operation + takes and gives a message of ``temporalio.api.streamservice.v1``. The bodies of the Records arrive as payloads that the SDK already encoded with the data converter and the payload codec, so a Store keeps payloads and not the user's types. A failed call raises diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index ab1c74073..4b1be2d12 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -86,5 +86,6 @@ of Records. It has three other costs. - A replay on a Worker with no cache reads the Store again on every Workflow Task. An append of Workflow code writes as the Workflow, with its Workflow Task as the epoch, and -records a marker with no body. A Store written in Python has no Rust runtime. On such a Store -an append runs in the Store's Local Activity, as before, and a read fails the Workflow. +records a marker with no body. A Store isn't meant to be written in Python, since such a Store +has no Rust runtime. On one, an append of Workflow code runs in the Store's Local Activity, as +before, and a read fails the Workflow. From 6c76edca01a5b60636099bbb26d3852756c25ba1 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 11:21:01 -0700 Subject: [PATCH 22/59] Moved Core to the Direct close of a Stream. Core now closes a Stream for Workflow code after History has the task that asked for it. The bridge protos and the payload visitor follow its new command and job. --- temporalio/bridge/_visitor.py | 10 ++ .../bridge/proto/external_data/__init__.py | 2 + .../proto/external_data/external_data_pb2.py | 18 ++- .../proto/external_data/external_data_pb2.pyi | 77 ++++++++- .../proto/workflow_activation/__init__.py | 2 + .../workflow_activation_pb2.py | 118 +++++++------- .../workflow_activation_pb2.pyi | 40 +++++ .../proto/workflow_commands/__init__.py | 2 + .../workflow_commands_pb2.py | 146 +++++++++--------- .../workflow_commands_pb2.pyi | 49 ++++++ temporalio/bridge/sdk-core | 2 +- temporalio/worker/_command_aware_visitor.py | 10 +- 12 files changed, 334 insertions(+), 142 deletions(-) diff --git a/temporalio/bridge/_visitor.py b/temporalio/bridge/_visitor.py index ada3052de..54238f891 100644 --- a/temporalio/bridge/_visitor.py +++ b/temporalio/bridge/_visitor.py @@ -334,6 +334,12 @@ async def _visit_coresdk_workflow_activation_ResolveStreamAppend( if o.HasField("failed"): await self._visit_temporal_api_failure_v1_Failure(fs, o.failed) + async def _visit_coresdk_workflow_activation_ResolveStreamClose( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("failed"): + await self._visit_temporal_api_failure_v1_Failure(fs, o.failed) + async def _visit_coresdk_workflow_activation_WorkflowActivationJob( self, fs: VisitorFunctions, o: Any ): @@ -387,6 +393,10 @@ async def _visit_coresdk_workflow_activation_WorkflowActivationJob( await self._visit_coresdk_workflow_activation_ResolveStreamAppend( fs, o.resolve_stream_append ) + elif o.HasField("resolve_stream_close"): + await self._visit_coresdk_workflow_activation_ResolveStreamClose( + fs, o.resolve_stream_close + ) async def _visit_coresdk_workflow_activation_WorkflowActivation( self, fs: VisitorFunctions, o: Any diff --git a/temporalio/bridge/proto/external_data/__init__.py b/temporalio/bridge/proto/external_data/__init__.py index 2230b1cbb..ffe40588e 100644 --- a/temporalio/bridge/proto/external_data/__init__.py +++ b/temporalio/bridge/proto/external_data/__init__.py @@ -2,6 +2,7 @@ LocalActivityMarkerData, PatchedMarkerData, StreamAppendEntry, + StreamCloseEntry, StreamMarkerData, StreamReadEnded, StreamReadEntry, @@ -11,6 +12,7 @@ "LocalActivityMarkerData", "PatchedMarkerData", "StreamAppendEntry", + "StreamCloseEntry", "StreamMarkerData", "StreamReadEnded", "StreamReadEntry", diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.py b/temporalio/bridge/proto/external_data/external_data_pb2.py index 188b86456..c2123054a 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.py +++ b/temporalio/bridge/proto/external_data/external_data_pb2.py @@ -27,7 +27,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a%temporal/sdk/core/common/common.proto"\xb2\x02\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x10\x61\x63tivation_index\x18\x08 \x01(\x04H\x00\x88\x01\x01\x42\x13\n\x11_activation_index"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\x84\x01\n\x10StreamMarkerData\x12\x35\n\x05reads\x18\x01 \x03(\x0b\x32&.coresdk.external_data.StreamReadEntry\x12\x39\n\x07\x61ppends\x18\x02 \x03(\x0b\x32(.coresdk.external_data.StreamAppendEntry"\xcb\x02\n\x0fStreamReadEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12\x14\n\x0c\x61\x66ter_offset\x18\x04 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x05 \x01(\x0c\x12\r\n\x05\x63ount\x18\x06 \x01(\r\x12\x0e\n\x06\x64igest\x18\x07 \x01(\x0c\x12\x35\n\x07records\x18\x08 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x13\n\x0bpage_counts\x18\t \x03(\r\x12\x18\n\x10page_activations\x18\n \x03(\r\x12\x33\n\x03\x65nd\x18\x0b \x01(\x0b\x32&.coresdk.external_data.StreamReadEnded"\xa9\x01\n\x0fStreamReadEnded\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x05 \x01(\r"\xc5\x01\n\x11StreamAppendEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x14\n\x0c\x66irst_offset\x18\x03 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x04 \x01(\x0c\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\rB2\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' + b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a%temporal/sdk/core/common/common.proto"\xb2\x02\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x10\x61\x63tivation_index\x18\x08 \x01(\x04H\x00\x88\x01\x01\x42\x13\n\x11_activation_index"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xbd\x01\n\x10StreamMarkerData\x12\x35\n\x05reads\x18\x01 \x03(\x0b\x32&.coresdk.external_data.StreamReadEntry\x12\x39\n\x07\x61ppends\x18\x02 \x03(\x0b\x32(.coresdk.external_data.StreamAppendEntry\x12\x37\n\x06\x63loses\x18\x03 \x03(\x0b\x32\'.coresdk.external_data.StreamCloseEntry"\xcb\x02\n\x0fStreamReadEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12\x14\n\x0c\x61\x66ter_offset\x18\x04 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x05 \x01(\x0c\x12\r\n\x05\x63ount\x18\x06 \x01(\r\x12\x0e\n\x06\x64igest\x18\x07 \x01(\x0c\x12\x35\n\x07records\x18\x08 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x13\n\x0bpage_counts\x18\t \x03(\r\x12\x18\n\x10page_activations\x18\n \x03(\r\x12\x33\n\x03\x65nd\x18\x0b \x01(\x0b\x32&.coresdk.external_data.StreamReadEnded"\xa9\x01\n\x0fStreamReadEnded\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x05 \x01(\r"\xc5\x01\n\x11StreamAppendEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x14\n\x0c\x66irst_offset\x18\x03 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x04 \x01(\x0c\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\r"\xec\x01\n\x10StreamCloseEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12?\n\x0cintent_epoch\x18\x03 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch\x12\x10\n\x08resolved\x18\x04 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\rB2\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -42,11 +42,13 @@ _PATCHEDMARKERDATA._serialized_start = 568 _PATCHEDMARKERDATA._serialized_end = 619 _STREAMMARKERDATA._serialized_start = 622 - _STREAMMARKERDATA._serialized_end = 754 - _STREAMREADENTRY._serialized_start = 757 - _STREAMREADENTRY._serialized_end = 1088 - _STREAMREADENDED._serialized_start = 1091 - _STREAMREADENDED._serialized_end = 1260 - _STREAMAPPENDENTRY._serialized_start = 1263 - _STREAMAPPENDENTRY._serialized_end = 1460 + _STREAMMARKERDATA._serialized_end = 811 + _STREAMREADENTRY._serialized_start = 814 + _STREAMREADENTRY._serialized_end = 1145 + _STREAMREADENDED._serialized_start = 1148 + _STREAMREADENDED._serialized_end = 1317 + _STREAMAPPENDENTRY._serialized_start = 1320 + _STREAMAPPENDENTRY._serialized_end = 1517 + _STREAMCLOSEENTRY._serialized_start = 1520 + _STREAMCLOSEENTRY._serialized_end = 1756 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.pyi b/temporalio/bridge/proto/external_data/external_data_pb2.pyi index d2e5f4788..840663e40 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.pyi +++ b/temporalio/bridge/proto/external_data/external_data_pb2.pyi @@ -157,6 +157,7 @@ class StreamMarkerData(google.protobuf.message.Message): READS_FIELD_NUMBER: builtins.int APPENDS_FIELD_NUMBER: builtins.int + CLOSES_FIELD_NUMBER: builtins.int @property def reads( self, @@ -169,14 +170,24 @@ class StreamMarkerData(google.protobuf.message.Message): ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ global___StreamAppendEntry ]: ... + @property + def closes( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamCloseEntry + ]: ... def __init__( self, *, reads: collections.abc.Iterable[global___StreamReadEntry] | None = ..., appends: collections.abc.Iterable[global___StreamAppendEntry] | None = ..., + closes: collections.abc.Iterable[global___StreamCloseEntry] | None = ..., ) -> None: ... def ClearField( - self, field_name: typing.Literal["appends", b"appends", "reads", b"reads"] + self, + field_name: typing.Literal[ + "appends", b"appends", "closes", b"closes", "reads", b"reads" + ], ) -> None: ... global___StreamMarkerData = StreamMarkerData @@ -387,3 +398,67 @@ class StreamAppendEntry(google.protobuf.message.Message): ) -> None: ... global___StreamAppendEntry = StreamAppendEntry + +@typing.final +class StreamCloseEntry(google.protobuf.message.Message): + """A close in one workflow task: its intent, in the task that asked for it, or its result, in the + task whose activation got it. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + INTENT_EPOCH_FIELD_NUMBER: builtins.int + RESOLVED_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + ACTIVATION_FIELD_NUMBER: builtins.int + seq: builtins.int + resolved: builtins.bool + """True on the result. `failure` is unset when the Stream is closed.""" + activation: builtins.int + """The activation in the task that got the result.""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def intent_epoch(self) -> temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch: + """Set on the intent, the epoch that Core closes with after the server took the task.""" + + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + intent_epoch: temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch + | None = ..., + resolved: builtins.bool = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + activation: builtins.int = ..., + ) -> None: ... + def HasField( + self, + field_name: typing.Literal[ + "failure", b"failure", "intent_epoch", b"intent_epoch", "stream", b"stream" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "activation", + b"activation", + "failure", + b"failure", + "intent_epoch", + b"intent_epoch", + "resolved", + b"resolved", + "seq", + b"seq", + "stream", + b"stream", + ], + ) -> None: ... + +global___StreamCloseEntry = StreamCloseEntry diff --git a/temporalio/bridge/proto/workflow_activation/__init__.py b/temporalio/bridge/proto/workflow_activation/__init__.py index fb214394a..4e459cc9b 100644 --- a/temporalio/bridge/proto/workflow_activation/__init__.py +++ b/temporalio/bridge/proto/workflow_activation/__init__.py @@ -17,6 +17,7 @@ ResolveRequestCancelExternalWorkflow, ResolveSignalExternalWorkflow, ResolveStreamAppend, + ResolveStreamClose, ResolveStreamRead, SignalWorkflow, StreamAppended, @@ -45,6 +46,7 @@ "ResolveRequestCancelExternalWorkflow", "ResolveSignalExternalWorkflow", "ResolveStreamAppend", + "ResolveStreamClose", "ResolveStreamRead", "SignalWorkflow", "StreamAppended", diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py index cac6b3961..863fbc082 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py @@ -52,7 +52,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\x82\x0c\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12M\n\x13resolve_stream_read\x18\x11 \x01(\x0b\x32..coresdk.workflow_activation.ResolveStreamReadH\x00\x12Q\n\x15resolve_stream_append\x18\x12 \x01(\x0b\x32\x30.coresdk.workflow_activation.ResolveStreamAppendH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\xe3\x0b\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x12\x65\n&propagated_nexus_serialization_context\x18\x1b \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"\xb1\x01\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause"\xb8\x01\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause"\xb2\x03\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x12\x65\n&propagated_nexus_serialization_context\x18\x08 \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\n"\x90\x01\n\x11ResolveStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x35\n\x07records\x18\x02 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x37\n\x03\x65nd\x18\x03 \x01(\x0b\x32*.coresdk.workflow_activation.StreamReadEnd"\x93\x01\n\rStreamReadEnd\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\xa1\x01\n\x13ResolveStreamAppend\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12?\n\x08\x61ppended\x18\x02 \x01(\x0b\x32+.coresdk.workflow_activation.StreamAppendedH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status";\n\x0eStreamAppended\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x02 \x01(\x0c\x42\x38\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' + b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\xd3\x0c\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12M\n\x13resolve_stream_read\x18\x11 \x01(\x0b\x32..coresdk.workflow_activation.ResolveStreamReadH\x00\x12Q\n\x15resolve_stream_append\x18\x12 \x01(\x0b\x32\x30.coresdk.workflow_activation.ResolveStreamAppendH\x00\x12O\n\x14resolve_stream_close\x18\x13 \x01(\x0b\x32/.coresdk.workflow_activation.ResolveStreamCloseH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\xe3\x0b\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x12\x65\n&propagated_nexus_serialization_context\x18\x1b \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"\xb1\x01\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause"\xb8\x01\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause"\xb2\x03\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x12\x65\n&propagated_nexus_serialization_context\x18\x08 \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\n"\x90\x01\n\x11ResolveStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x35\n\x07records\x18\x02 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x37\n\x03\x65nd\x18\x03 \x01(\x0b\x32*.coresdk.workflow_activation.StreamReadEnd"\x93\x01\n\rStreamReadEnd\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\xa1\x01\n\x13ResolveStreamAppend\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12?\n\x08\x61ppended\x18\x02 \x01(\x0b\x32+.coresdk.workflow_activation.StreamAppendedH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status";\n\x0eStreamAppended\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x02 \x01(\x0c"S\n\x12ResolveStreamClose\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x30\n\x06\x66\x61iled\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.FailureB8\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -73,61 +73,63 @@ _WORKFLOWACTIVATION._serialized_start = 649 _WORKFLOWACTIVATION._serialized_end = 1195 _WORKFLOWACTIVATIONJOB._serialized_start = 1198 - _WORKFLOWACTIVATIONJOB._serialized_end = 2736 - _INITIALIZEWORKFLOW._serialized_start = 2739 - _INITIALIZEWORKFLOW._serialized_end = 4246 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 4167 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 4246 - _FIRETIMER._serialized_start = 4248 - _FIRETIMER._serialized_end = 4272 - _RESOLVEACTIVITY._serialized_start = 4274 - _RESOLVEACTIVITY._serialized_end = 4383 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4386 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4723 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4725 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4784 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4787 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 4953 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 4955 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 5051 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 5053 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 5158 - _UPDATERANDOMSEED._serialized_start = 5160 - _UPDATERANDOMSEED._serialized_end = 5203 - _QUERYWORKFLOW._serialized_start = 5206 - _QUERYWORKFLOW._serialized_end = 5466 - _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 4167 - _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 4246 - _CANCELWORKFLOW._serialized_start = 5468 - _CANCELWORKFLOW._serialized_end = 5500 - _SIGNALWORKFLOW._serialized_start = 5503 - _SIGNALWORKFLOW._serialized_end = 5792 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 4167 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 4246 - _NOTIFYHASPATCH._serialized_start = 5794 - _NOTIFYHASPATCH._serialized_end = 5828 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5831 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 6008 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 6011 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 6195 - _DOUPDATE._serialized_start = 6198 - _DOUPDATE._serialized_end = 6632 - _DOUPDATE_HEADERSENTRY._serialized_start = 4167 - _DOUPDATE_HEADERSENTRY._serialized_end = 4246 - _RESOLVENEXUSOPERATIONSTART._serialized_start = 6635 - _RESOLVENEXUSOPERATIONSTART._serialized_end = 6789 - _RESOLVENEXUSOPERATION._serialized_start = 6791 - _RESOLVENEXUSOPERATION._serialized_end = 6880 - _REMOVEFROMCACHE._serialized_start = 6883 - _REMOVEFROMCACHE._serialized_end = 7235 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 6997 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 7235 - _RESOLVESTREAMREAD._serialized_start = 7238 - _RESOLVESTREAMREAD._serialized_end = 7382 - _STREAMREADEND._serialized_start = 7385 - _STREAMREADEND._serialized_end = 7532 - _RESOLVESTREAMAPPEND._serialized_start = 7535 - _RESOLVESTREAMAPPEND._serialized_end = 7696 - _STREAMAPPENDED._serialized_start = 7698 - _STREAMAPPENDED._serialized_end = 7757 + _WORKFLOWACTIVATIONJOB._serialized_end = 2817 + _INITIALIZEWORKFLOW._serialized_start = 2820 + _INITIALIZEWORKFLOW._serialized_end = 4327 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 4248 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 4327 + _FIRETIMER._serialized_start = 4329 + _FIRETIMER._serialized_end = 4353 + _RESOLVEACTIVITY._serialized_start = 4355 + _RESOLVEACTIVITY._serialized_end = 4464 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4467 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4804 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4806 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4865 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4868 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 5034 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 5036 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 5132 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 5134 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 5239 + _UPDATERANDOMSEED._serialized_start = 5241 + _UPDATERANDOMSEED._serialized_end = 5284 + _QUERYWORKFLOW._serialized_start = 5287 + _QUERYWORKFLOW._serialized_end = 5547 + _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 4248 + _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 4327 + _CANCELWORKFLOW._serialized_start = 5549 + _CANCELWORKFLOW._serialized_end = 5581 + _SIGNALWORKFLOW._serialized_start = 5584 + _SIGNALWORKFLOW._serialized_end = 5873 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 4248 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 4327 + _NOTIFYHASPATCH._serialized_start = 5875 + _NOTIFYHASPATCH._serialized_end = 5909 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5912 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 6089 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 6092 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 6276 + _DOUPDATE._serialized_start = 6279 + _DOUPDATE._serialized_end = 6713 + _DOUPDATE_HEADERSENTRY._serialized_start = 4248 + _DOUPDATE_HEADERSENTRY._serialized_end = 4327 + _RESOLVENEXUSOPERATIONSTART._serialized_start = 6716 + _RESOLVENEXUSOPERATIONSTART._serialized_end = 6870 + _RESOLVENEXUSOPERATION._serialized_start = 6872 + _RESOLVENEXUSOPERATION._serialized_end = 6961 + _REMOVEFROMCACHE._serialized_start = 6964 + _REMOVEFROMCACHE._serialized_end = 7316 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 7078 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 7316 + _RESOLVESTREAMREAD._serialized_start = 7319 + _RESOLVESTREAMREAD._serialized_end = 7463 + _STREAMREADEND._serialized_start = 7466 + _STREAMREADEND._serialized_end = 7613 + _RESOLVESTREAMAPPEND._serialized_start = 7616 + _RESOLVESTREAMAPPEND._serialized_end = 7777 + _STREAMAPPENDED._serialized_start = 7779 + _STREAMAPPENDED._serialized_end = 7838 + _RESOLVESTREAMCLOSE._serialized_start = 7840 + _RESOLVESTREAMCLOSE._serialized_end = 7923 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi index 8886676e0..de21dd362 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi @@ -247,6 +247,7 @@ class WorkflowActivationJob(google.protobuf.message.Message): RESOLVE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int RESOLVE_STREAM_READ_FIELD_NUMBER: builtins.int RESOLVE_STREAM_APPEND_FIELD_NUMBER: builtins.int + RESOLVE_STREAM_CLOSE_FIELD_NUMBER: builtins.int REMOVE_FROM_CACHE_FIELD_NUMBER: builtins.int @property def initialize_workflow(self) -> global___InitializeWorkflow: @@ -329,6 +330,10 @@ class WorkflowActivationJob(google.protobuf.message.Message): def resolve_stream_append(self) -> global___ResolveStreamAppend: """An append to a Stream resolved.""" + @property + def resolve_stream_close(self) -> global___ResolveStreamClose: + """A close of a Stream resolved.""" + @property def remove_from_cache(self) -> global___RemoveFromCache: """Remove the workflow identified by the [WorkflowActivation] containing this job from the @@ -360,6 +365,7 @@ class WorkflowActivationJob(google.protobuf.message.Message): resolve_nexus_operation: global___ResolveNexusOperation | None = ..., resolve_stream_read: global___ResolveStreamRead | None = ..., resolve_stream_append: global___ResolveStreamAppend | None = ..., + resolve_stream_close: global___ResolveStreamClose | None = ..., remove_from_cache: global___RemoveFromCache | None = ..., ) -> None: ... def HasField( @@ -395,6 +401,8 @@ class WorkflowActivationJob(google.protobuf.message.Message): b"resolve_signal_external_workflow", "resolve_stream_append", b"resolve_stream_append", + "resolve_stream_close", + b"resolve_stream_close", "resolve_stream_read", b"resolve_stream_read", "signal_workflow", @@ -438,6 +446,8 @@ class WorkflowActivationJob(google.protobuf.message.Message): b"resolve_signal_external_workflow", "resolve_stream_append", b"resolve_stream_append", + "resolve_stream_close", + b"resolve_stream_close", "resolve_stream_read", b"resolve_stream_read", "signal_workflow", @@ -469,6 +479,7 @@ class WorkflowActivationJob(google.protobuf.message.Message): "resolve_nexus_operation", "resolve_stream_read", "resolve_stream_append", + "resolve_stream_close", "remove_from_cache", ] | None @@ -1816,3 +1827,32 @@ class StreamAppended(google.protobuf.message.Message): ) -> None: ... global___StreamAppended = StreamAppended + +@typing.final +class ResolveStreamClose(google.protobuf.message.Message): + """The result of a `CloseStream`, in a workflow task after the one that asked for it.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + FAILED_FIELD_NUMBER: builtins.int + seq: builtins.int + """The sequence number of the `CloseStream`""" + @property + def failed(self) -> temporalio.api.failure.v1.message_pb2.Failure: + """Unset when the Stream is closed, also when it was closed already.""" + + def __init__( + self, + *, + seq: builtins.int = ..., + failed: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failed", b"failed"] + ) -> builtins.bool: ... + def ClearField( + self, field_name: typing.Literal["failed", b"failed", "seq", b"seq"] + ) -> None: ... + +global___ResolveStreamClose = ResolveStreamClose diff --git a/temporalio/bridge/proto/workflow_commands/__init__.py b/temporalio/bridge/proto/workflow_commands/__init__.py index 92fcecf8b..15dde24ee 100644 --- a/temporalio/bridge/proto/workflow_commands/__init__.py +++ b/temporalio/bridge/proto/workflow_commands/__init__.py @@ -5,6 +5,7 @@ CancelSignalWorkflow, CancelTimer, CancelWorkflowExecution, + CloseStream, CloseStreamRead, CompleteWorkflowExecution, ContinueAsNewWorkflowExecution, @@ -36,6 +37,7 @@ "CancelSignalWorkflow", "CancelTimer", "CancelWorkflowExecution", + "CloseStream", "CloseStreamRead", "CompleteWorkflowExecution", "ContinueAsNewWorkflowExecution", diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py index f06e2de58..1908ea226 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py @@ -49,7 +49,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a$temporal/api/stream/v1/message.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\x82\x12\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x12G\n\x11start_stream_read\x18\x17 \x01(\x0b\x32*.coresdk.workflow_commands.StartStreamReadH\x00\x12G\n\x11\x63lose_stream_read\x18\x18 \x01(\x0b\x32*.coresdk.workflow_commands.CloseStreamReadH\x00\x12\x45\n\x10\x61ppend_to_stream\x18\x19 \x01(\x0b\x32).coresdk.workflow_commands.AppendToStreamH\x00\x42\t\n\x07variant"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x93\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12#\n\x1binclude_arguments_in_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xe1\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12I\n\x13versioning_override\x18\x15 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x87\x02\n\x0fStartStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12*\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12/\n\x0cidle_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bmax_records\x18\x06 \x01(\r\x12\x11\n\tmax_bytes\x18\x07 \x01(\x04\x12\x1a\n\x12records_in_history\x18\x08 \x01(\x08"\x1e\n\x0f\x43loseStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xc7\x01\n\x0e\x41ppendToStream\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12/\n\x06\x62odies\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x32\n\x08producer\x18\x04 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12\x10\n\x08sequence\x18\x05 \x01(\x03*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' + b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a$temporal/api/stream/v1/message.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\xc2\x12\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x12G\n\x11start_stream_read\x18\x17 \x01(\x0b\x32*.coresdk.workflow_commands.StartStreamReadH\x00\x12G\n\x11\x63lose_stream_read\x18\x18 \x01(\x0b\x32*.coresdk.workflow_commands.CloseStreamReadH\x00\x12\x45\n\x10\x61ppend_to_stream\x18\x19 \x01(\x0b\x32).coresdk.workflow_commands.AppendToStreamH\x00\x12>\n\x0c\x63lose_stream\x18\x1a \x01(\x0b\x32&.coresdk.workflow_commands.CloseStreamH\x00\x42\t\n\x07variant"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x93\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12#\n\x1binclude_arguments_in_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xe1\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12I\n\x13versioning_override\x18\x15 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x87\x02\n\x0fStartStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12*\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12/\n\x0cidle_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bmax_records\x18\x06 \x01(\r\x12\x11\n\tmax_bytes\x18\x07 \x01(\x04\x12\x1a\n\x12records_in_history\x18\x08 \x01(\x08"\x1e\n\x0f\x43loseStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xc7\x01\n\x0e\x41ppendToStream\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12/\n\x06\x62odies\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x32\n\x08producer\x18\x04 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12\x10\n\x08sequence\x18\x05 \x01(\x03"\x81\x01\n\x0b\x43loseStream\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x32\n\x08producer\x18\x03 \x01(\x0b\x32 .temporal.api.stream.v1.Producer*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -75,76 +75,78 @@ _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_options = b"8\001" _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._options = None _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_options = b"8\001" - _ACTIVITYCANCELLATIONTYPE._serialized_start = 9540 - _ACTIVITYCANCELLATIONTYPE._serialized_end = 9628 + _ACTIVITYCANCELLATIONTYPE._serialized_start = 9736 + _ACTIVITYCANCELLATIONTYPE._serialized_end = 9824 _WORKFLOWCOMMAND._serialized_start = 596 - _WORKFLOWCOMMAND._serialized_end = 2902 - _STARTTIMER._serialized_start = 2904 - _STARTTIMER._serialized_end = 2987 - _CANCELTIMER._serialized_start = 2989 - _CANCELTIMER._serialized_end = 3015 - _SCHEDULEACTIVITY._serialized_start = 3018 - _SCHEDULEACTIVITY._serialized_end = 3842 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3763 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3842 - _SCHEDULELOCALACTIVITY._serialized_start = 3845 - _SCHEDULELOCALACTIVITY._serialized_end = 4632 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3763 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3842 - _REQUESTCANCELACTIVITY._serialized_start = 4634 - _REQUESTCANCELACTIVITY._serialized_end = 4670 - _REQUESTCANCELLOCALACTIVITY._serialized_start = 4672 - _REQUESTCANCELLOCALACTIVITY._serialized_end = 4713 - _QUERYRESULT._serialized_start = 4716 - _QUERYRESULT._serialized_end = 4872 - _QUERYSUCCESS._serialized_start = 4874 - _QUERYSUCCESS._serialized_end = 4939 - _COMPLETEWORKFLOWEXECUTION._serialized_start = 4941 - _COMPLETEWORKFLOWEXECUTION._serialized_end = 5017 - _FAILWORKFLOWEXECUTION._serialized_start = 5019 - _FAILWORKFLOWEXECUTION._serialized_end = 5093 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 5096 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 6010 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5853 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5929 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3763 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3842 - _CANCELWORKFLOWEXECUTION._serialized_start = 6012 - _CANCELWORKFLOWEXECUTION._serialized_end = 6088 - _SETPATCHMARKER._serialized_start = 6090 - _SETPATCHMARKER._serialized_end = 6144 - _STARTCHILDWORKFLOWEXECUTION._serialized_start = 6147 - _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7396 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3763 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3842 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5853 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5929 - _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7398 - _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7472 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7475 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7617 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7620 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 8019 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3763 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3842 - _CANCELSIGNALWORKFLOW._serialized_start = 8021 - _CANCELSIGNALWORKFLOW._serialized_end = 8056 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 8058 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 8159 - _MODIFYWORKFLOWPROPERTIES._serialized_start = 8161 - _MODIFYWORKFLOWPROPERTIES._serialized_end = 8240 - _UPDATERESPONSE._serialized_start = 8243 - _UPDATERESPONSE._serialized_end = 8453 - _SCHEDULENEXUSOPERATION._serialized_start = 8456 - _SCHEDULENEXUSOPERATION._serialized_end = 8994 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 8944 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 8994 - _REQUESTCANCELNEXUSOPERATION._serialized_start = 8996 - _REQUESTCANCELNEXUSOPERATION._serialized_end = 9038 - _STARTSTREAMREAD._serialized_start = 9041 - _STARTSTREAMREAD._serialized_end = 9304 - _CLOSESTREAMREAD._serialized_start = 9306 - _CLOSESTREAMREAD._serialized_end = 9336 - _APPENDTOSTREAM._serialized_start = 9339 - _APPENDTOSTREAM._serialized_end = 9538 + _WORKFLOWCOMMAND._serialized_end = 2966 + _STARTTIMER._serialized_start = 2968 + _STARTTIMER._serialized_end = 3051 + _CANCELTIMER._serialized_start = 3053 + _CANCELTIMER._serialized_end = 3079 + _SCHEDULEACTIVITY._serialized_start = 3082 + _SCHEDULEACTIVITY._serialized_end = 3906 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3827 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3906 + _SCHEDULELOCALACTIVITY._serialized_start = 3909 + _SCHEDULELOCALACTIVITY._serialized_end = 4696 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3827 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3906 + _REQUESTCANCELACTIVITY._serialized_start = 4698 + _REQUESTCANCELACTIVITY._serialized_end = 4734 + _REQUESTCANCELLOCALACTIVITY._serialized_start = 4736 + _REQUESTCANCELLOCALACTIVITY._serialized_end = 4777 + _QUERYRESULT._serialized_start = 4780 + _QUERYRESULT._serialized_end = 4936 + _QUERYSUCCESS._serialized_start = 4938 + _QUERYSUCCESS._serialized_end = 5003 + _COMPLETEWORKFLOWEXECUTION._serialized_start = 5005 + _COMPLETEWORKFLOWEXECUTION._serialized_end = 5081 + _FAILWORKFLOWEXECUTION._serialized_start = 5083 + _FAILWORKFLOWEXECUTION._serialized_end = 5157 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 5160 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 6074 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5917 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5993 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3827 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3906 + _CANCELWORKFLOWEXECUTION._serialized_start = 6076 + _CANCELWORKFLOWEXECUTION._serialized_end = 6152 + _SETPATCHMARKER._serialized_start = 6154 + _SETPATCHMARKER._serialized_end = 6208 + _STARTCHILDWORKFLOWEXECUTION._serialized_start = 6211 + _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7460 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3827 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3906 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5917 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5993 + _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7462 + _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7536 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7539 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7681 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7684 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 8083 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3827 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3906 + _CANCELSIGNALWORKFLOW._serialized_start = 8085 + _CANCELSIGNALWORKFLOW._serialized_end = 8120 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 8122 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 8223 + _MODIFYWORKFLOWPROPERTIES._serialized_start = 8225 + _MODIFYWORKFLOWPROPERTIES._serialized_end = 8304 + _UPDATERESPONSE._serialized_start = 8307 + _UPDATERESPONSE._serialized_end = 8517 + _SCHEDULENEXUSOPERATION._serialized_start = 8520 + _SCHEDULENEXUSOPERATION._serialized_end = 9058 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 9008 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 9058 + _REQUESTCANCELNEXUSOPERATION._serialized_start = 9060 + _REQUESTCANCELNEXUSOPERATION._serialized_end = 9102 + _STARTSTREAMREAD._serialized_start = 9105 + _STARTSTREAMREAD._serialized_end = 9368 + _CLOSESTREAMREAD._serialized_start = 9370 + _CLOSESTREAMREAD._serialized_end = 9400 + _APPENDTOSTREAM._serialized_start = 9403 + _APPENDTOSTREAM._serialized_end = 9602 + _CLOSESTREAM._serialized_start = 9605 + _CLOSESTREAM._serialized_end = 9734 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi index 4c9e7bb63..9b7f6aec4 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi @@ -109,6 +109,7 @@ class WorkflowCommand(google.protobuf.message.Message): START_STREAM_READ_FIELD_NUMBER: builtins.int CLOSE_STREAM_READ_FIELD_NUMBER: builtins.int APPEND_TO_STREAM_FIELD_NUMBER: builtins.int + CLOSE_STREAM_FIELD_NUMBER: builtins.int @property def user_metadata(self) -> temporalio.api.sdk.v1.user_metadata_pb2.UserMetadata: """User metadata that may or may not be persisted into history depending on the command type. @@ -191,6 +192,8 @@ class WorkflowCommand(google.protobuf.message.Message): def close_stream_read(self) -> global___CloseStreamRead: ... @property def append_to_stream(self) -> global___AppendToStream: ... + @property + def close_stream(self) -> global___CloseStream: ... def __init__( self, *, @@ -232,6 +235,7 @@ class WorkflowCommand(google.protobuf.message.Message): start_stream_read: global___StartStreamRead | None = ..., close_stream_read: global___CloseStreamRead | None = ..., append_to_stream: global___AppendToStream | None = ..., + close_stream: global___CloseStream | None = ..., ) -> None: ... def HasField( self, @@ -246,6 +250,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"cancel_timer", "cancel_workflow_execution", b"cancel_workflow_execution", + "close_stream", + b"close_stream", "close_stream_read", b"close_stream_read", "complete_workflow_execution", @@ -305,6 +311,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"cancel_timer", "cancel_workflow_execution", b"cancel_workflow_execution", + "close_stream", + b"close_stream", "close_stream_read", b"close_stream_read", "complete_workflow_execution", @@ -382,6 +390,7 @@ class WorkflowCommand(google.protobuf.message.Message): "start_stream_read", "close_stream_read", "append_to_stream", + "close_stream", ] | None ): ... @@ -2121,3 +2130,43 @@ class AppendToStream(google.protobuf.message.Message): ) -> None: ... global___AppendToStream = AppendToStream + +@typing.final +class CloseStream(google.protobuf.message.Message): + """Close a Stream. Core records the intent in this workflow task, closes the Stream after the + server took the task, and gives the result in a `ResolveStreamClose` job of a later task. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + PRODUCER_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's incremental sequence number of Stream closes, used as the close's identifier""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def producer(self) -> temporalio.api.stream.v1.message_pb2.Producer: + """The workflow's Producer, of kind workflow, with its id and run id and no epoch. Core sets the + epoch to the workflow task that recorded the close. + """ + + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + producer: temporalio.api.stream.v1.message_pb2.Producer | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["producer", b"producer", "stream", b"stream"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "producer", b"producer", "seq", b"seq", "stream", b"stream" + ], + ) -> None: ... + +global___CloseStream = CloseStream diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 0754feced..230783854 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 0754feced59bce59b1b3fa85258bb306c3f18abe +Subproject commit 230783854ad543dcb98812203f3ee11fe05a4795 diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index 282b2d547..058d80cd7 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -17,6 +17,7 @@ ResolveRequestCancelExternalWorkflow, ResolveSignalExternalWorkflow, ResolveStreamAppend, + ResolveStreamClose, ResolveStreamRead, StreamReadEnd, ) @@ -103,8 +104,8 @@ async def _visit_coresdk_workflow_activation_ResolveStreamRead( with current_command(CommandType.COMMAND_TYPE_UNSPECIFIED, o.seq, stream=True): await super()._visit_coresdk_workflow_activation_ResolveStreamRead(fs, o) - # Core builds the failure of a Stream read or append itself, with no codec, so no pass - # may decode it. + # Core builds the failure of a Stream read, append or close itself, with no codec, so no + # pass may decode it. async def _visit_coresdk_workflow_activation_StreamReadEnd( self, fs: VisitorFunctions, o: StreamReadEnd ) -> None: @@ -115,6 +116,11 @@ async def _visit_coresdk_workflow_activation_ResolveStreamAppend( ) -> None: pass + async def _visit_coresdk_workflow_activation_ResolveStreamClose( + self, fs: VisitorFunctions, o: ResolveStreamClose + ) -> None: + pass + # Workflow commands with payloads async def _visit_coresdk_workflow_commands_CompleteWorkflowExecution( self, fs: VisitorFunctions, o: CompleteWorkflowExecution From 8849be48c2e367d2eb23f38a9801887ca2b7c08f Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 11:24:19 -0700 Subject: [PATCH 23/59] Made Core close a Workflow's Stream after the task that asked for it. The crate refuses a Workflow close with no epoch, and a close in a failed task must close nothing. Core records the intent and closes after the server took the task. --- temporalio/worker/_stream_store.py | 87 ++++++++++++++++++----- temporalio/worker/_workflow_instance.py | 36 +++++++++- tests/nexus/test_stream_store_workflow.py | 47 +++++++++--- 3 files changed, 141 insertions(+), 29 deletions(-) diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index 87407705f..7c56c953e 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -8,6 +8,7 @@ from datetime import timedelta from typing import TYPE_CHECKING, Any +import google.protobuf.message from nexusrpc import OutputT import temporalio.api.failure.v1 @@ -118,25 +119,74 @@ async def result() -> OutputT: ) -def _start_replayed_direct_append( # pyright: ignore[reportUnusedFunction] +def _start_direct_close( instance: _WorkflowInstanceImpl, input: StartNexusOperationInput[Any, OutputT], -) -> temporalio.workflow.NexusOperationHandle[OutputT] | None: - """Replay a Direct append on a Worker with no Stream config, or give ``None``. + request: temporalio.api.streamservice.v1.CloseStreamRequest, + store_name: str, +) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Send a close of a Workflow Producer to Core. - The marker of a Direct append holds its result, so its replay needs no Store. A ref - that names no Store sends an empty name, and Core takes the Store of History. + Core records the intent in this Workflow Task, closes the Stream after the server took + the task, and gives the result in a later task. So a failed task closes nothing. """ - if input.operation_name != "AppendRecords": - return None payload_converter, _ = instance._system_nexus_converters(input) - request = payload_converter._to_transfer_value(input.input) + close = temporalio.bridge.proto.workflow_commands.CloseStream( + stream=temporalio.api.stream.v1.StreamRef(id=request.ref.id, store=store_name), + producer=request.producer, + ) + close.producer.workflow.ClearField("epoch") + closed = instance._start_stream_close(close) + + async def result() -> OutputT: + await closed + return payload_converter._from_transfer_value( + temporalio.api.streamservice.v1.CloseStreamResponse(), input.output_type + ) + + return _StreamStoreOperationHandle( + instance.create_task(result(), name=f"stream store: {input.operation_name}") + ) + + +def _start_direct_call( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: google.protobuf.message.Message, + store_name: str, +) -> temporalio.workflow.NexusOperationHandle[OutputT] | None: + """Send an append or a close of a Workflow Producer to Core, or give ``None``. + + Core makes these calls itself, with the Workflow Task as the epoch of the Producer. + A Producer of another kind, and every other operation, keep the Activity path. + """ + if ( + isinstance(request, temporalio.api.streamservice.v1.AppendRecordsRequest) + and request.producer.WhichOneof("kind") == "workflow" + ): + return _start_direct_append(instance, input, request, store_name) if ( - not isinstance(request, temporalio.api.streamservice.v1.AppendRecordsRequest) - or request.producer.WhichOneof("kind") != "workflow" + isinstance(request, temporalio.api.streamservice.v1.CloseStreamRequest) + and request.producer.WhichOneof("kind") == "workflow" ): + return _start_direct_close(instance, input, request, store_name) + return None + + +def _start_replayed_direct_call( # pyright: ignore[reportUnusedFunction] + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], +) -> temporalio.workflow.NexusOperationHandle[OutputT] | None: + """Replay a Direct append or close on a Worker with no Stream config, or give ``None``. + + The marker of a Direct call holds its result, so its replay needs no Store. A ref that + names no Store sends an empty name, and Core takes the Store of History. + """ + if input.operation_name not in ("AppendRecords", "CloseStream"): return None - return _start_direct_append(instance, input, request, request.ref.store) + payload_converter, _ = instance._system_nexus_converters(input) + request = payload_converter._to_transfer_value(input.input) + return _start_direct_call(instance, input, request, request.ref.store) def _read_command( # pyright: ignore[reportUnusedFunction] @@ -239,14 +289,13 @@ def start_operation( # have raises `StreamConfigError` here, with no I/O, and fails the Workflow Task. store = self.config._store_for(ref, _workflow_target()) options = self.config.workflow_call_options - if ( - isinstance(request, temporalio.api.streamservice.v1.AppendRecordsRequest) - and isinstance(store, _BridgeStreamStore) - and request.producer.WhichOneof("kind") == "workflow" - ): - # Core appends inside the Workflow Task and sets the epoch of the Producer - # to that task. A Producer of another kind keeps the Local Activity path. - return _start_direct_append(instance, input, request, store.name) + if isinstance(store, _BridgeStreamStore): + # Core appends and closes for a Workflow Producer itself, with the Workflow + # Task as its epoch. A Producer of another kind keeps the Local Activity path. + # Core orders the calls of one Stream, so a Direct call skips the order below. + direct = _start_direct_call(instance, input, request, store.name) + if direct is not None: + return direct tails = instance._stream_write_tails tail_key = (store.name, ref.id) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index fd9eb5e75..72d668245 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -101,7 +101,7 @@ from ._stream_store import ( _no_stream_config_error, _read_command, - _start_replayed_direct_append, + _start_replayed_direct_call, _stream_error_from_core, _WorkflowStreamStore, ) @@ -380,6 +380,7 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: int, asyncio.Future[temporalio.bridge.proto.workflow_activation.StreamAppended], ] = {} + self._pending_stream_closes: dict[int, asyncio.Future[None]] = {} # Reads that Workflow code closed. A job of one of them can still come in the # activation that the close answers, and is dropped. self._closed_stream_reads: set[int] = set() @@ -716,6 +717,8 @@ def _apply( self._apply_resolve_stream_read(job.resolve_stream_read) elif job.HasField("resolve_stream_append"): self._apply_resolve_stream_append(job.resolve_stream_append) + elif job.HasField("resolve_stream_close"): + self._apply_resolve_stream_close(job.resolve_stream_close) elif job.HasField("signal_workflow"): self._apply_signal_workflow(job.signal_workflow) elif job.HasField("initialize_workflow"): @@ -1019,6 +1022,25 @@ def _apply_resolve_stream_append( ) fut.set_exception(error) + def _apply_resolve_stream_close( + self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamClose + ) -> None: + fut = self._pending_stream_closes.pop(job.seq, None) + if fut is None: + raise RuntimeError(f"Failed finding Stream close for sequence {job.seq}") + # Code that stopped waiting for the close has cancelled the future. + if fut.done(): + return + if not job.HasField("failed"): + fut.set_result(None) + return + error: BaseException | None = _stream_error_from_core(job.failed) + if error is None: + error = self._context_free_failure_converter.from_failure( + job.failed, self._context_free_payload_converter + ) + fut.set_exception(error) + def _apply_resolve_activity( self, job: temporalio.bridge.proto.workflow_activation.ResolveActivity ) -> None: @@ -1897,6 +1919,16 @@ def _start_stream_append( self._pending_stream_appends[append.seq] = fut return fut + def _start_stream_close( + self, close: temporalio.bridge.proto.workflow_commands.CloseStream + ) -> asyncio.Future[None]: + command = self._add_command() + close.seq = self._next_seq("stream_close") + command.close_stream.CopyFrom(close) + fut: asyncio.Future[None] = self.create_future() + self._pending_stream_closes[close.seq] = fut + return fut + def workflow_close_stream_read( self, reading: temporalio.nexus.system._support.StreamReading[Any], @@ -2500,7 +2532,7 @@ async def _schedule_system_nexus_operation( store = self._stream_store if store is None: replayed = ( - _start_replayed_direct_append(self, input) + _start_replayed_direct_call(self, input) if self._is_replaying else None ) diff --git a/tests/nexus/test_stream_store_workflow.py b/tests/nexus/test_stream_store_workflow.py index c6633ebe0..5e952060d 100644 --- a/tests/nexus/test_stream_store_workflow.py +++ b/tests/nexus/test_stream_store_workflow.py @@ -334,10 +334,14 @@ async def append_records( class CloseRequestStore(RecordingStreamStore): - """Keeps each close request.""" + """Keeps each close request. + + It wraps a Store written in Python, so a close of Workflow code runs in its Local + Activity. + """ def __init__(self) -> None: - super().__init__(MemoryStreamStore()) + super().__init__(DictStreamStore()) self.closes: list[temporalio.api.streamservice.v1.CloseStreamRequest] = [] async def close_stream( @@ -1190,7 +1194,8 @@ async def test_failed_workflow_task_after_close_leaves_the_stream_closed( ): global _fail_close_task _fail_close_task = True - stream_client = _with_store(client, MemoryStreamStore()) + # A Store written in Python, since Core closes a Stream on a Store with a runtime. + stream_client = _with_store(client, DictStreamStore()) stream = await stream_client.create_stream( id=f"stream-{uuid.uuid4()}", body_type=str ) @@ -1218,9 +1223,36 @@ async def test_failed_workflow_task_after_close_leaves_the_stream_closed( ) +async def test_failed_workflow_task_after_direct_close_leaves_the_stream_open( + client: Client, +): + global _fail_close_task + _fail_close_task = True + stream_client = _with_store(client, MemoryStreamStore()) + stream = await stream_client.create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + async with new_worker(stream_client, CloseThenFailTaskOnceWorkflow) as worker: + result = await stream_client.execute_workflow( + CloseThenFailTaskOnceWorkflow.run, + stream.ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + # Core makes the close with no Activity. The failed task recorded no intent, so it + # closed nothing, and the retried task appends again before it closes the Stream. + assert result == "closed" + records = await _read_all(stream_client, stream.ref.id) + assert [record.body for record in records] == ["a", "a"] + assert await _describe_status(stream_client, stream.ref.id) == ( + StreamStatus.StreamStatusClosed + ) + + async def test_activity_interceptors_see_the_close_activity(client: Client): interceptor = RecordingInterceptor() - stream_client = _with_store(client, MemoryStreamStore()) + # A Store written in Python, since Core closes a Stream on a Store with a runtime. + stream_client = _with_store(client, DictStreamStore()) async with new_worker( stream_client, CloseWorkflow, interceptors=[interceptor] ) as worker: @@ -1230,10 +1262,9 @@ async def test_activity_interceptors_see_the_close_activity(client: Client): id=f"workflow-{uuid.uuid4()}", task_queue=worker.task_queue, ) - # The create and the close each run as a Local Activity. Core makes the append in - # the Workflow Task, with no Activity. - assert interceptor.activity_types == ["__temporal_stream_store_call:memory"] * 2 - assert interceptor.is_local == [True] * 2 + # The create, the append and the close each run as a Local Activity. + assert interceptor.activity_types == ["__temporal_stream_store_call:dict"] * 3 + assert interceptor.is_local == [True] * 3 class SlowFirstAppendStore(LostAnswerStore): From b4fed0813b24f595bb14a8643a9c4053ff8bc48a Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 11:24:20 -0700 Subject: [PATCH 24/59] Documented the Direct close in two Workflow Tasks. Code must await its close, since a close that it doesn't await is lost if the Worker dies when the Workflow ends. An append and a close of Workflow code carry the task as their epoch. --- changelog/added/patient-river-heron.md | 1 + temporalio/client/_stream_store.py | 26 +++++++++++-------- temporalio/contrib/streams/README.md | 36 +++++++++++++++++++------- temporalio/worker/_replayer.py | 16 ++++++------ 4 files changed, 51 insertions(+), 28 deletions(-) diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md index 0c31d9fbe..ccf1a0ec1 100644 --- a/changelog/added/patient-river-heron.md +++ b/changelog/added/patient-river-heron.md @@ -1,2 +1,3 @@ Add `iter` to the Workflow `Stream`, so Workflow code reads each Record of a Stream as it arrives. Core reads the Store inside the Workflow Task, and a read costs one marker in History for each Workflow Task that it spans. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). A Workflow's append on the Redis Store or `MemoryStreamStore` now runs in Core inside the Workflow Task, with no Local Activity. It writes as the Workflow, with its Workflow Task as the epoch. +A Workflow's close on the Redis Store or `MemoryStreamStore` now runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close: a close that the code doesn't await is lost if the Worker dies when the Workflow ends. An append and a close of Workflow code always carry the Workflow Task as their epoch. diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index c14dbb41a..ca9546eb8 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -65,8 +65,8 @@ class StreamStoreCallOptions: Workflow code can't do I/O, so the default :py:meth:`StreamStore.call_from_workflow` calls its Store from a Local Activity. - Core makes a read and a Workflow's append on a Store with a Rust runtime itself, with - no Activity, so these options don't apply to them. + Core makes a read, and a Workflow's append and close, on a Store with a Rust runtime + itself, with no Activity, so these options don't apply to them. A :py:class:`StreamConfig` holds them in :py:attr:`StreamConfig.workflow_call_options`, and every Store of that config @@ -109,8 +109,8 @@ class StreamStore(ABC): subclassed in Python. A Store that is written in Python works for a client and an Activity, for example as a double in a test. Workflow code loses the Direct calls on it. An append of Workflow code then runs in a Local Activity, with one marker in - History for each call, and ``iter`` fails the Workflow, so Workflow code can't read - its Streams. + History for each call, a close runs in a regular Activity after its Workflow Task, + and ``iter`` fails the Workflow, so Workflow code can't read its Streams. Such a Store implements :py:attr:`name` and the five operations. Each operation takes and gives a message of @@ -201,9 +201,9 @@ async def call_from_workflow( can't append to it. On a Store with a Rust runtime, such as the Redis Store and - :py:class:`temporalio.testing.MemoryStreamStore`, Core appends for Workflow code - inside the Workflow Task, and this method gets no append of a Producer of kind - Workflow. + :py:class:`temporalio.testing.MemoryStreamStore`, Core appends and closes for + Workflow code itself, and this method gets no append and no close of a Producer + of kind Workflow. Args: operation: The Nexus operation name, ``CreateStream``, ``AppendRecords``, @@ -494,10 +494,14 @@ class StreamConfig: Task. The appends and the close of one Workflow on one Stream run one at a time, in the order of the calls. - Workflow code calls a Store from a Local Activity, also for a close. So it needs - no Worker that polls for Activity Tasks. A Local Activity reaches the Store before - its Workflow Task completes. If that task fails after a close, the Stream stays - closed. The retry of the task then can't append to it. + On a Store with a Rust runtime, Core appends and closes for Workflow code itself, + with no Activity. Core closes the Stream only after the server took the Workflow Task + of the close, so a failed task closes nothing. On a Store written in Python, and for + a caller's Producer of another kind, Workflow code calls the Store from a Local + Activity, also for a close. So it needs no Worker that polls for Activity Tasks. A + Local Activity reaches the Store before its Workflow Task completes. If that task + fails after a close, the Stream stays closed. The retry of the task then can't append + to it. .. warning:: This API is experimental and subject to change. diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 4b1be2d12..0a636d8f7 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -45,9 +45,9 @@ replacement of the first attempt's Records. ## Reads and appends in Workflow code -The helpers run on the Streams API, so they follow how Workflow code reads and appends. On a -Store with a Rust runtime, such as the Redis Store or `MemoryStreamStore`, Core reads and -appends a Stream itself, inside the Workflow Task. A read is `iter` of the Workflow `Stream`. +The helpers run on the Streams API, so they follow how Workflow code reads, appends and +closes. On a Store with a Rust runtime, such as the Redis Store or `MemoryStreamStore`, Core +reads, appends and closes a Stream itself, with no Activity. A read is `iter` of the Workflow `Stream`. It gives each Record as it arrives and ends at a stop. ```python @@ -67,8 +67,7 @@ while True: `max_bytes`. After the loop, `stop` tells which stop ended the read, and `done` is true when the Stream is closed and the read gave its last Record. That stop is `Done` also when the last Record filled a cap. The idle time counts only while the read waits for the Store. A -new read from `last_offset` loses -and repeats no Record, also after `aclose` or a `break`. A read that the code leaves with +new read from `last_offset` loses and repeats no Record, also after `aclose` or a `break`. A read that the code leaves with `break` and doesn't close runs on in Core until one of its stops. A read costs one marker in History for each Workflow Task that it spans, whatever the number @@ -85,7 +84,26 @@ of Records. It has three other costs. 256 KiB. - A replay on a Worker with no cache reads the Store again on every Workflow Task. -An append of Workflow code writes as the Workflow, with its Workflow Task as the epoch, and -records a marker with no body. A Store isn't meant to be written in Python, since such a Store -has no Rust runtime. On one, an append of Workflow code runs in the Store's Local Activity, as -before, and a read fails the Workflow. +An append and a close of Workflow code always write as the Workflow, with the Workflow Task as +their epoch. An append runs inside its Workflow Task and records a marker with no body. + +A close spans two Workflow Tasks. The task that calls `close` records only the intent. Core +closes the Stream after the server took that task, and the next task gives the result. So a +failed task closes nothing, and its retry can still append. A Workflow that awaits its close +never completes in the task that asked for it. Await the close, also in a `finally` block. + +```python +stream = await workflow.create_stream(id=stream_id, body_type=str) +try: + await stream.append("a") +finally: + await stream.close() +``` + +A close that the code doesn't await still happens when the Workflow ends in the same task, +after the server took that task. If the Worker dies there, that close is lost, and the Stream +stays open. + +A Store isn't meant to be written in Python, since such a Store has no Rust runtime. On one, +an append and a close of Workflow code run in the Store's Local Activity, and a read fails the +Workflow. diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index a7c55438d..1078b4ccf 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -68,14 +68,14 @@ def __init__( ``stream_config`` takes the Stream config of the client that the worker had, so a replay makes the same commands. A history that records a - create, a close, a describe, or an append through a Store's Local Activity - needs it. With no Stream config, Workflow code gets an error in place of that - call, and the replay fails with a nondeterminism error. A replay calls no - method of its Stores from Python. An append that Core made in the Workflow - Task needs no Stream config, and neither does a read of Workflow code with - ``iter`` that ran under the option ``HISTORY`` of ``workflow_read_replay_from``. - That holds also when the ref names no Store, since Core takes the Store from the - history. A read that ran under the default, ``StreamReplaySource.STORE``, has no + create, a describe, or an append or a close through a Store's Local Activity needs + it. With no Stream config, Workflow code gets an error in place of that call, + and the replay fails with a nondeterminism error. A replay calls no method of + its Stores from Python. An append or a close that Core made needs no Stream + config, and a replay closes nothing. A read of Workflow code with ``iter`` that + ran under the option ``HISTORY`` of ``workflow_read_replay_from`` needs none + either. That holds also when the ref names no Store, since Core takes the Store + from the history. A read that ran under the default, ``StreamReplaySource.STORE``, has no Records in the history. Core reads them from the Store again, so the replay needs this config and Stores that still have the Records. Without them the replay fails with a :py:class:`RuntimeError` whose message has From 9dab5da3f185b195a76b8afe45f8eb8cd15e0255 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 11:26:09 -0700 Subject: [PATCH 25/59] Tested the Direct close of a Stream in Workflow code. The tests cover the two tasks of an awaited close, a close in `finally`, a failed task, a close not awaited, a cancel, replays with and without a config, and a Store in Python. --- tests/nexus/test_stream_direct.py | 279 ++++++++++++++++++++++++++++++ 1 file changed, 279 insertions(+) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 6ad7689f7..7d02f4e6f 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -14,6 +14,7 @@ from temporalio import activity, workflow from temporalio.api.common.v1 import Payload from temporalio.api.enums.v1 import EventType, StreamErrorReason +from temporalio.bridge.proto.external_data import StreamCloseEntry, StreamMarkerData from temporalio.client import ( Client, StreamConfig, @@ -36,6 +37,7 @@ ProducerKindWorkflow, StreamRecord, StreamRef, + StreamStatus, ) from temporalio.testing import MemoryStreamStore from temporalio.worker import Replayer, Worker @@ -1418,3 +1420,280 @@ async def test_worker_without_workflows_builds_no_store_runtime(client: Client): task_queue=f"tq-{uuid.uuid4()}", workflows=[AppendOneWorkflow], ) + + +# The Direct close. + + +def _close_entries(history: WorkflowHistory) -> list[StreamCloseEntry]: + """Give the close entries of the `core_stream` markers, in order.""" + entries: list[StreamCloseEntry] = [] + for event in history.events: + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + entries.extend(StreamMarkerData.FromString(payload.data).closes) + return entries + + +def _scheduled_activities(history: WorkflowHistory) -> list[str]: + return [ + event.activity_task_scheduled_event_attributes.activity_type.name + for event in history.events + if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED + ] + + +async def _status(client: Client, ref: StreamRef[str]) -> StreamStatus | None: + return (await client.get_stream(ref, body_type=str).describe()).status + + +@workflow.defn +class AppendMaybeCloseWorkflow: + @workflow.run + async def run(self, stream_id: str, close: bool) -> None: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + if close: + await stream.close() + + +async def test_awaited_close_closes_in_one_more_task(client: Client): + stream_client = _stream_client(client) + closed_ref = await _filled_stream(stream_client, [], close=False) + open_ref = await _filled_stream(stream_client, [], close=False) + closed, _ = await _run(stream_client, AppendMaybeCloseWorkflow, closed_ref.id, True) + left_open, _ = await _run( + stream_client, AppendMaybeCloseWorkflow, open_ref.id, False + ) + assert await _status(stream_client, closed_ref) == StreamStatus.StreamStatusClosed + assert await _status(stream_client, open_ref) == StreamStatus.StreamStatusOpen + history = await closed.fetch_history() + assert _scheduled_activities(history) == [] + assert "core_local_activity" not in _markers(history) + # The task of the close records its intent, and the next one gives the result. + assert _completed_tasks(history) == ( + _completed_tasks(await left_open.fetch_history()) + 1 + ) + intent, result = _close_entries(history) + assert intent.HasField("intent_epoch") and not intent.resolved + assert result.resolved and not result.HasField("failure") + + +@workflow.defn +class FinallyCloseWorkflow: + @workflow.run + async def run(self, stream_id: str, fail: bool) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + try: + await stream.append("a") + if fail: + raise ApplicationError("failed on purpose", non_retryable=True) + return "returned" + finally: + await stream.close() + + +@pytest.mark.parametrize("fail", [False, True]) +async def test_close_in_finally(client: Client, fail: bool): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + if fail: + with pytest.raises(WorkflowFailureError) as err: + await _run(stream_client, FinallyCloseWorkflow, ref.id, True) + assert isinstance(err.value.cause, ApplicationError) + assert err.value.cause.message == "failed on purpose" + else: + _, result = await _run(stream_client, FinallyCloseWorkflow, ref.id, False) + assert result == "returned" + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + + +@workflow.defn +class AppendAfterCloseWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + await stream.close() + try: + await stream.append("b") + except StreamError as err: + return [StreamErrorReason.Name(err.reason), str(err.non_retryable)] + return ["appended"] + + +async def test_append_after_the_close_is_refused(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + _, result = await _run(stream_client, AppendAfterCloseWorkflow, ref.id) + assert result == ["STREAM_ERROR_REASON_CLOSED", "True"] + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + + +_fail_close_task = True + + +@workflow.defn(sandboxed=False) +class FailAfterCloseOnceWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + close = asyncio.create_task(stream.close()) + # A few turns of the loop let the close send its command in this activation. + for _ in range(10): + await asyncio.sleep(0) + global _fail_close_task + if _fail_close_task: + _fail_close_task = False + raise RuntimeError("Failed the Workflow Task once") + await close + + +async def test_failed_task_after_the_close_leaves_the_stream_open(client: Client): + global _fail_close_task + _fail_close_task = True + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run(stream_client, FailAfterCloseOnceWorkflow, ref.id) + assert not _fail_close_task + # The retry could append, so the failed task closed nothing. + assert [record.body for record in await _read_all(stream_client, ref)] == ["a", "a"] + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + # History has one intent and one result, from the retry. + entries = _close_entries(await handle.fetch_history()) + assert [entry.resolved for entry in entries] == [False, True] + + +@workflow.defn +class CloseTwiceWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.close() + await stream.close() + return "closed" + + +async def test_close_of_a_closed_stream_succeeds(client: Client): + stream_client = _stream_client(client) + # A client closed this Stream before the Workflow closes it twice. + ref = await _filled_stream(stream_client, ["a"]) + _, result = await _run(stream_client, CloseTwiceWorkflow, ref.id) + assert result == "closed" + open_ref = await _filled_stream(stream_client, ["a"], close=False) + _, result = await _run(stream_client, CloseTwiceWorkflow, open_ref.id) + assert result == "closed" + assert await _status(stream_client, open_ref) == StreamStatus.StreamStatusClosed + + +async def test_close_without_a_cache_and_in_replays(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run( + stream_client, AppendMaybeCloseWorkflow, ref.id, True, max_cached_workflows=0 + ) + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + history = await handle.fetch_history() + assert [entry.resolved for entry in _close_entries(history)] == [False, True] + # A Store with the same Stream, still open, shows that a replay closes nothing. + replay_store = MemoryStreamStore() + replay_client = _stream_client(client, replay_store) + await replay_client.create_stream(id=ref.id, body_type=str) + await Replayer( + workflows=[AppendMaybeCloseWorkflow], + stream_config=replay_client.config()["stream_config"], + ).replay_workflow(history) + assert await _status(replay_client, ref) == StreamStatus.StreamStatusOpen + # The ref names no Store, so a Replayer with no Stream config takes it from History. + await Replayer(workflows=[AppendMaybeCloseWorkflow]).replay_workflow(history) + + +@workflow.defn +class CloseNotAwaitedWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + # The task sends the close command, and the Workflow returns at once. + asyncio.create_task(stream.close()) + await asyncio.sleep(0) + return "returned" + + +async def test_close_not_awaited_closes_after_the_workflow_ends(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + handle, result = await _run(stream_client, CloseNotAwaitedWorkflow, ref.id) + assert result == "returned" + + async def closed() -> bool: + return await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + + for _ in range(50): + if await closed(): + break + await asyncio.sleep(0.1) + assert await closed() + # The task that ends the Workflow has the intent, and no task gives a result. + entries = _close_entries(await handle.fetch_history()) + assert [entry.resolved for entry in entries] == [False] + + +@activity.defn +async def sleep_a_while() -> None: + await asyncio.sleep(2) + + +@workflow.defn +class CancelDuringCloseWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + # The Local Activity holds the task of the close open, so the cancel comes in the + # task that gives the result of the close. + workflow.start_local_activity( + sleep_a_while, start_to_close_timeout=timedelta(seconds=10) + ) + try: + await stream.close() + except asyncio.CancelledError: + return "cancelled" + return "closed" + + +async def test_cancel_while_the_close_waits(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker( + stream_client, CancelDuringCloseWorkflow, activities=[sleep_a_while] + ) as worker: + handle = await stream_client.start_workflow( + CancelDuringCloseWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await asyncio.sleep(0.5) + await handle.cancel() + result = await asyncio.wait_for(handle.result(), timeout=30) + # The code gets the cancel, and Core still closed the Stream after the task of the + # intent. + assert result == "cancelled" + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + + +async def test_python_store_closes_through_its_activity(client: Client): + stream_client = _stream_client(client, DictStreamStore()) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run(stream_client, AppendMaybeCloseWorkflow, ref.id, True) + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + history = await handle.fetch_history() + assert _scheduled_activities(history) == ["__temporal_stream_store_call:dict"] + assert _close_entries(history) == [] From d922e85e270f9f51229aebb4b07c6996d551b537 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 11:29:07 -0700 Subject: [PATCH 26/59] Wrapped two lines of the Streams README. The edit for the Direct close left two lines over 100 characters. --- temporalio/contrib/streams/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 0a636d8f7..99d6a07b3 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -47,8 +47,8 @@ replacement of the first attempt's Records. The helpers run on the Streams API, so they follow how Workflow code reads, appends and closes. On a Store with a Rust runtime, such as the Redis Store or `MemoryStreamStore`, Core -reads, appends and closes a Stream itself, with no Activity. A read is `iter` of the Workflow `Stream`. -It gives each Record as it arrives and ends at a stop. +reads, appends and closes a Stream itself, with no Activity. A read is `iter` of the Workflow +`Stream`. It gives each Record as it arrives and ends at a stop. ```python tokens = workflow.get_stream(tokens_ref, body_type=Token) @@ -67,8 +67,9 @@ while True: `max_bytes`. After the loop, `stop` tells which stop ended the read, and `done` is true when the Stream is closed and the read gave its last Record. That stop is `Done` also when the last Record filled a cap. The idle time counts only while the read waits for the Store. A -new read from `last_offset` loses and repeats no Record, also after `aclose` or a `break`. A read that the code leaves with -`break` and doesn't close runs on in Core until one of its stops. +new read from `last_offset` loses and repeats no Record, also after `aclose` or a `break`. A +read that the code leaves with `break` and doesn't close runs on in Core until one of its +stops. A read costs one marker in History for each Workflow Task that it spans, whatever the number of Records. It has three other costs. From 95c7cf3ff8be0410e0058e6db86c5709d20c442d Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 12:11:01 -0700 Subject: [PATCH 27/59] Moved Core to its Stream tests that wait for events. Core changed only its own tests there, so the bridge is the same. --- temporalio/bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 230783854..0ce78463c 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 230783854ad543dcb98812203f3ee11fe05a4795 +Subproject commit 0ce78463cb0816604f7c95aed098e05783ab7ecd From 1738876e58c8f927c9bc3b81f1581b2516915cea Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 22:07:34 -0700 Subject: [PATCH 28/59] Gave a read of Workflow code no idle bound when `idle_timeout` is None. Core now reads an unset idle timeout as no bound and a zero as no wait. So `None` sends nothing, and a zero goes to Core as a zero, with no 1 ms in its place. --- temporalio/bridge/sdk-core | 2 +- temporalio/worker/_stream_store.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 0ce78463c..879a0f603 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 0ce78463cb0816604f7c95aed098e05783ab7ecd +Subproject commit 879a0f6039f1771129838b45b344312cecfd5cf7 diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index 7c56c953e..836f0506e 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -5,7 +5,6 @@ import asyncio import dataclasses from collections.abc import Generator -from datetime import timedelta from typing import TYPE_CHECKING, Any import google.protobuf.message @@ -207,11 +206,9 @@ def _read_command( # pyright: ignore[reportUnusedFunction] if options.timeout is not None: start.timeout.FromTimedelta(options.timeout) if options.idle_timeout is not None: - # Core reads a zero idle timeout as unset, which is 1 s. The least wait that Core - # keeps stands for no wait. - start.idle_timeout.FromTimedelta( - options.idle_timeout or timedelta(milliseconds=1) - ) + # Core reads an unset idle timeout as no bound and a zero as no wait, so only a + # value that the code gave goes in the command. + start.idle_timeout.FromTimedelta(options.idle_timeout) if options.max_records is not None: start.max_records = options.max_records max_bytes = options.max_bytes From 20d27ae744c0af5563f3e1a36df091764ee0705a Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 22:07:41 -0700 Subject: [PATCH 29/59] Documented that a read of Workflow code waits for the Stream's close. A read with no `idle_timeout` holds its Workflow Task open on a quiet Stream, with one task about every 8 s. The docs say so, and show one loop as the plain example. --- changelog/added/patient-river-heron.md | 2 +- temporalio/contrib/streams/README.md | 24 +++++++++++++++----- temporalio/nexus/system/_support.py | 31 ++++++++++++++++++++------ 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md index ccf1a0ec1..8821f42f6 100644 --- a/changelog/added/patient-river-heron.md +++ b/changelog/added/patient-river-heron.md @@ -1,3 +1,3 @@ -Add `iter` to the Workflow `Stream`, so Workflow code reads each Record of a Stream as it arrives. Core reads the Store inside the Workflow Task, and a read costs one marker in History for each Workflow Task that it spans. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). +Add `iter` to the Workflow `Stream`, so Workflow code reads each Record of a Stream as it arrives. By default a read has no bound and reads until the Stream is closed. A read that waits on a quiet Stream holds its Workflow Task open, with one Workflow Task about every 8 s, and an `idle_timeout` ends it with `Idle`. Core reads the Store inside the Workflow Task, and a read costs one marker in History for each Workflow Task that it spans. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). A Workflow's append on the Redis Store or `MemoryStreamStore` now runs in Core inside the Workflow Task, with no Local Activity. It writes as the Workflow, with its Workflow Task as the epoch. A Workflow's close on the Redis Store or `MemoryStreamStore` now runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close: a close that the code doesn't await is lost if the Worker dies when the Workflow ends. An append and a close of Workflow code always carry the Workflow Task as their epoch. diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 99d6a07b3..a109a34e9 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -48,23 +48,35 @@ replacement of the first attempt's Records. The helpers run on the Streams API, so they follow how Workflow code reads, appends and closes. On a Store with a Rust runtime, such as the Redis Store or `MemoryStreamStore`, Core reads, appends and closes a Stream itself, with no Activity. A read is `iter` of the Workflow -`Stream`. It gives each Record as it arrives and ends at a stop. +`Stream`. It gives each Record as it arrives and ends at a stop. By default a read has no +bound, so it reads until the Stream is closed and the read gave its last Record. ```python tokens = workflow.get_stream(tokens_ref, body_type=Token) shown = workflow.get_stream(shown_ref, body_type=str) +async for record in tokens.iter(): + await shown.append(record.body.text) +``` + +`iter` takes `from_offset` and four bounds, `timeout`, `idle_timeout`, `max_records` and +`max_bytes`. Each bound is `None` by default, which is no bound. An `idle_timeout` ends a +read with `Idle` when no Record comes for that long, and zero gives what is in the Stream +and doesn't wait. After a bound ended a read, a new read from `last_offset` goes on where it +stopped. This loop waits with a timer while the Stream is quiet. + +```python last = None while True: - reading = tokens.iter(from_offset=last) + reading = tokens.iter(from_offset=last, idle_timeout=timedelta(seconds=5)) async for record in reading: await shown.append(record.body.text) last = reading.last_offset if reading.done: break + await workflow.sleep(timedelta(minutes=1)) ``` -`iter` takes `timeout`, `from_offset`, `idle_timeout` (1 s by default), `max_records` and -`max_bytes`. After the loop, `stop` tells which stop ended the read, and `done` is true when +After the loop, `stop` tells which stop ended the read, and `done` is true when the Stream is closed and the read gave its last Record. That stop is `Done` also when the last Record filled a cap. The idle time counts only while the read waits for the Store. A new read from `last_offset` loses and repeats no Record, also after `aclose` or a `break`. A @@ -76,7 +88,9 @@ of Records. It has three other costs. - A read keeps its Workflow Task open. A Signal, an Update, a cancel or an Activity result reaches the Workflow at the next Workflow Task, at the heartbeat of the task at the latest. - That is about 8 s with the default task timeout of 10 s. + That is about 8 s with the default task timeout of 10 s. A read that waits on a quiet + Stream holds its task open the whole time, with one Workflow Task about every 8 s. An + `idle_timeout` ends such a read with `Idle`. - By default, `StreamReplaySource.STORE`, the marker holds a range of offsets, a count and a digest, and no Record. A replay reads the Store again, so the Records must stay in the Store for as long as the Workflow can replay. The option diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index 3e82b67b7..38c8560ec 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -201,7 +201,11 @@ class StreamReadStop(enum.IntEnum): """The next Record would pass ``max_bytes``.""" Idle = 4 - """No Record came for ``idle_timeout`` while the read waited for the Store.""" + """No Record came for ``idle_timeout`` while the read waited for the Store. + + Only a read with an ``idle_timeout`` ends with this stop. A read with none waits on a + quiet Stream until another stop holds. + """ Timeout = 5 """``timeout`` passed since the read started.""" @@ -372,7 +376,8 @@ def iter_stream( It is the ``iter`` of a Workflow Stream. It does no I/O and sends nothing, so it takes no ``await``. The read starts at the first step of the ``async for`` loop. Core then reads the Store inside the Workflow Task and gives the Records to the loop - as they arrive. The loop ends at the first stop. Then + as they arrive. By default a read has no bound, so it reads until the Stream is + closed and the read gave its last Record. The loop ends at the first stop. Then :py:attr:`StreamReading.stop` tells which stop it was, and :py:attr:`StreamReading.done` is true when the Stream is closed and the read gave its last Record. That stop is ``Done`` also when the last Record filled a cap. A new @@ -382,16 +387,25 @@ def iter_stream( step one read. Each Record goes to one of them, and both end when the read ends. A read that failed raises its error at every step, in each task. + .. code-block:: python + + async for record in tokens.iter(): + await shown.append(record.body.text) + + A cap, a ``timeout`` or an ``idle_timeout`` ends a read before the Stream is done. + A loop of reads from ``last_offset`` then goes on where the last read stopped. + .. code-block:: python last = None while True: - reading = tokens.iter(from_offset=last) + reading = tokens.iter(from_offset=last, idle_timeout=timedelta(seconds=5)) async for record in reading: await shown.append(record.body.text) last = reading.last_offset if reading.done: break + await workflow.sleep(timedelta(minutes=1)) A read costs one marker in History for each Workflow Task that it spans, whatever the number of Records. A replay gives the loop the same Records at the same points. @@ -400,7 +414,9 @@ def iter_stream( * A read keeps its Workflow Task open, as a Local Activity does. A Signal, an Update, a cancel or an Activity result reaches the Workflow at the next Workflow Task. That is when the read ends, or at the heartbeat of the task at the latest, - which is about 8 s with the default task timeout of 10 s. + which is about 8 s with the default task timeout of 10 s. A read that waits on a + quiet Stream holds its task open the whole time, with one Workflow Task about + every 8 s. An ``idle_timeout`` ends such a read with ``Idle``. * With the default :py:attr:`temporalio.client.StreamReplaySource.STORE`, History holds no Record. A replay reads the Records from the Store again, so they must stay in the Store for as long as the Workflow can replay. When they are gone, the @@ -426,9 +442,10 @@ def iter_stream( restart. ``None`` is no bound. from_offset: Read the Records after this offset. ``None`` is the start of the Stream. - idle_timeout: The read ends when no Record comes for this long while it waits for - the Store. Time in which its Records wait for the loop doesn't count. ``None`` - is 1 s. Zero gives what is in the Stream and doesn't wait. + idle_timeout: The read ends with ``Idle`` when no Record comes for this long + while it waits for the Store. Time in which its Records wait for the loop + doesn't count. ``None`` is no bound, so the read waits for the next Record. + Zero gives what is in the Stream and doesn't wait. max_records: The most Records of the read. ``None`` is no cap. max_bytes: The most bytes of the Records of the read, each as its encoded ``StreamRecord``. The first Record always comes. ``None`` is no cap, unless From 2f4d5ab90a446bb525f7161aab65b2028c93d459 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 22:07:41 -0700 Subject: [PATCH 30/59] Tested a read with no idle bound, and a zero idle timeout. Three Workflows dropped the outer loop that only the 1 s default needed. A plain `iter` now reads across a quiet gap, and a zero ends with `Idle` on an empty Stream. --- tests/contrib/redis/test_streams.py | 20 +++----- tests/nexus/test_stream_direct.py | 79 ++++++++++++++++++++++------- 2 files changed, 69 insertions(+), 30 deletions(-) diff --git a/tests/contrib/redis/test_streams.py b/tests/contrib/redis/test_streams.py index ed05b432d..ae23a9981 100644 --- a/tests/contrib/redis/test_streams.py +++ b/tests/contrib/redis/test_streams.py @@ -209,18 +209,14 @@ async def run(self, stream_id: str, seen_id: str) -> list[str]: stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) seen = workflow.get_stream(StreamRef(id=seen_id), body_type=str) got: list[str] = [] - last = None - while True: - reading = stream.iter(from_offset=last) - async for record in reading: - got.append(record.body) - if record.body == "b": - await seen.append("the first step") - last = reading.last_offset - if reading.done: - assert reading.stop is not None - got.append(reading.stop.name) - break + # A read has no idle bound, so one read waits across the Activity's two steps. + reading = stream.iter() + async for record in reading: + got.append(record.body) + if record.body == "b": + await seen.append("the first step") + assert reading.stop is not None + got.append(reading.stop.name) await writer return got diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 7d02f4e6f..3afd4feb3 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -168,14 +168,9 @@ async def stream_model_output(call: ModelCall) -> None: async def model_chunks(chunks_id: str) -> AsyncGenerator[str, None]: """Yield each chunk of the model's Stream, as the model stub of a library does.""" stream = workflow.get_stream(StreamRef(id=chunks_id), body_type=str) - last = None - while True: - reading = stream.iter(from_offset=last) - async for record in reading: - yield record.body - last = reading.last_offset - if reading.done: - return + # A read has no idle bound, so one read waits for the model until it closes the Stream. + async for record in stream.iter(): + yield record.body @workflow.defn @@ -356,9 +351,63 @@ async def test_read_with_zero_idle_timeout_gives_what_is_there(client: Client): assert result.bodies == ["a", "b"] assert (result.stop, result.done) == ("Idle", False) events = (await handle.fetch_history()).events - # The read waits for no new Record, so the run takes far less than the default 1 s. + # The read doesn't wait for the Store, so the run ends long before the first heartbeat. took = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() - assert took < timedelta(milliseconds=800) + assert took < timedelta(seconds=5) + + +async def test_read_with_zero_idle_timeout_ends_immediately_on_an_empty_stream( + client: Client, +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run( + stream_client, ReadOnceWorkflow, ReadArgs(ref.id, idle_timeout_ms=0) + ) + assert result.bodies == [] + assert (result.stop, result.done) == ("Idle", False) + assert result.last_offset is None + events = (await handle.fetch_history()).events + # An empty Stream gives no reason to wait either. + took = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert took < timedelta(seconds=5) + + +@activity.defn +async def append_after_a_quiet_gap(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + await asyncio.sleep(2) + await stream.append("b") + await stream.close() + + +@workflow.defn +class QuietGapWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[Any]: + writer = workflow.start_activity( + append_after_a_quiet_gap, + stream_id, + start_to_close_timeout=timedelta(seconds=30), + ) + reading = workflow.get_stream(StreamRef(id=stream_id), body_type=str).iter() + got = [record.body async for record in reading] + await writer + return [got, reading.stop and reading.stop.name, reading.done] + + +async def test_plain_iter_reads_across_a_quiet_gap(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + _, result = await _run( + stream_client, + QuietGapWorkflow, + ref.id, + activities=[append_after_a_quiet_gap], + ) + # One read gets both Records across the gap of 2 s, with no Idle stop in between. + assert result == [["a", "b"], "Done", True] # What `iter` checks and when the read starts. @@ -1109,14 +1158,8 @@ async def run(self, burst: Burst) -> int: ) stream = workflow.get_stream(StreamRef(id=burst.stream_id), body_type=str) count = 0 - last = None - while True: - reading = stream.iter(from_offset=last) - async for _ in reading: - count += 1 - last = reading.last_offset - if reading.done: - break + async for _ in stream.iter(): + count += 1 await writer return count From 48196e0a83f24ee614286382f64f9a4b3c96bc1f Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 22:44:24 -0700 Subject: [PATCH 31/59] Moved Core to a read that looks at the Store before its timeout. A read with a short timeout gave no Record when its Workflow Task started late, which failed a test on the cloud run. The stubs of the Core protos have the new comments of the read bounds. --- .../proto/workflow_commands/workflow_commands_pb2.pyi | 8 ++++++-- temporalio/bridge/sdk-core | 2 +- temporalio/nexus/system/_support.py | 4 +++- tests/nexus/test_stream_direct.py | 11 +++++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi index 9b7f6aec4..8e7292f5f 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi @@ -2001,11 +2001,15 @@ class StartStreamRead(google.protobuf.message.Message): def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... @property def timeout(self) -> google.protobuf.duration_pb2.Duration: - """The longest time the read may take. Unset means no bound.""" + """The longest time the read may take. Unset means no bound. A read that gave no Record yet + looks at the Store once before its timeout ends it. + """ @property def idle_timeout(self) -> google.protobuf.duration_pb2.Duration: - """The read ends when no Record comes for this long. Unset means one second.""" + """The read ends when no Record comes for this long. Unset means no bound. Zero gives the + Records that are in the Stream, and ends with no wait. + """ def __init__( self, diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 879a0f603..4ec830eec 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 879a0f6039f1771129838b45b344312cecfd5cf7 +Subproject commit 4ec830eec8cf34ba0f7c4e4c721cd318e6a320b9 diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index 38c8560ec..5025a088d 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -439,7 +439,9 @@ def iter_stream( stream: The Workflow Stream to read. timeout: The longest time that the read takes, from the time it starts. Its deadline holds across a replay, a failed Workflow Task and a Worker - restart. ``None`` is no bound. + restart. A read that gave no Record yet looks at the Store once before + its timeout ends it, so a short timeout still gives the Records that are + there. ``None`` is no bound. from_offset: Read the Records after this offset. ``None`` is the start of the Stream. idle_timeout: The read ends with ``Idle`` when no Record comes for this long diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 3afd4feb3..7685bfe50 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -302,6 +302,17 @@ async def test_read_stops_at_timeout(client: Client): assert (result.stop, result.done) == ("Timeout", False) +async def test_read_past_its_timeout_gives_what_is_there(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b"], close=False) + # The timeout passes before Core starts the read, which still looks at the Store once. + result = await _read_once( + stream_client, ReadArgs(ref.id, timeout_ms=1, idle_timeout_ms=30_000) + ) + assert result.bodies == ["a", "b"] + assert (result.stop, result.done) == ("Timeout", False) + + async def test_read_stops_at_max_records(client: Client): stream_client = _stream_client(client) ref = await _filled_stream(stream_client, ["a", "b", "c", "d"]) From edab0850d5d0abe6ec86d8f17488dc98e88dac7e Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 23:20:42 -0700 Subject: [PATCH 32/59] Moved Core to its Stream metrics and the shutdown wait for a close. Core now records metrics for the Direct calls, and a graceful shutdown waits for a close in flight. Its protos changed only in a comment that the generated files already had. --- temporalio/bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 4ec830eec..9d7e25131 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 4ec830eec8cf34ba0f7c4e4c721cd318e6a320b9 +Subproject commit 9d7e25131ec70c06cc6921f9c42896df4304ca0d From c2720e06e07199e64e2776e5b9e3911254696f80 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 23:20:42 -0700 Subject: [PATCH 33/59] Documented the Stream metrics and the shutdown wait for a close. A close that the code doesn't await is lost only when the Worker dies or the Store doesn't answer in 10 s. The README lists the metrics that Core records for the Direct calls. --- changelog/added/patient-river-heron.md | 3 ++- temporalio/contrib/streams/README.md | 25 +++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md index 8821f42f6..238a196a3 100644 --- a/changelog/added/patient-river-heron.md +++ b/changelog/added/patient-river-heron.md @@ -1,3 +1,4 @@ Add `iter` to the Workflow `Stream`, so Workflow code reads each Record of a Stream as it arrives. By default a read has no bound and reads until the Stream is closed. A read that waits on a quiet Stream holds its Workflow Task open, with one Workflow Task about every 8 s, and an `idle_timeout` ends it with `Idle`. Core reads the Store inside the Workflow Task, and a read costs one marker in History for each Workflow Task that it spans. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). A Workflow's append on the Redis Store or `MemoryStreamStore` now runs in Core inside the Workflow Task, with no Local Activity. It writes as the Workflow, with its Workflow Task as the epoch. -A Workflow's close on the Redis Store or `MemoryStreamStore` now runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close: a close that the code doesn't await is lost if the Worker dies when the Workflow ends. An append and a close of Workflow code always carry the Workflow Task as their epoch. +A Workflow's close on the Redis Store or `MemoryStreamStore` now runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close: a graceful shutdown of the Worker waits at most 10 s for a close that the code doesn't await, and a Worker that dies when the Workflow ends loses it. An append and a close of Workflow code always carry the Workflow Task as their epoch. +Core records metrics for the Direct calls of a Stream, such as `temporal_stream_read_total`, `temporal_stream_read_records` and `temporal_stream_close_latency`, each with the attribute `stream_store`. diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index a109a34e9..097690fa2 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -116,9 +116,30 @@ finally: ``` A close that the code doesn't await still happens when the Workflow ends in the same task, -after the server took that task. If the Worker dies there, that close is lost, and the Stream -stays open. +after the server took that task. A graceful shutdown of the Worker waits for such a close, for +at most 10 s. Only a Worker that dies there, or a Store that doesn't answer in that time, loses +the close, and the Stream stays open. A Store isn't meant to be written in Python, since such a Store has no Rust runtime. On one, an append and a close of Workflow code run in the Store's Local Activity, and a read fails the Workflow. + +## Metrics + +Core records these metrics for the Direct calls of Workflow code. Their prefix is the metric +prefix of the `Runtime`, which is `temporal_` by default. Each metric has the attribute +`stream_store` with the Store of the call, next to the `namespace`, `task_queue` and +`workflow_type` that a Local Activity metric has. A replay that takes a result from a marker +counts nothing. + +| Metric | What it counts | +| --- | --- | +| `temporal_stream_read_total` | Reads that Core started | +| `temporal_stream_read_failed` | Reads that ended with a failure | +| `temporal_stream_read_records` | Records that reads gave to Workflow code | +| `temporal_stream_append_total`, `_failed` | Appends that Core sent, and those that failed | +| `temporal_stream_append_latency` | From the start of an append in Core to its result | +| `temporal_stream_close_total`, `_failed` | Closes that Core sent, and those that failed | +| `temporal_stream_close_latency` | From the start of a close in Core to its result | +| `temporal_stream_replay_read_total`, `_failed` | Marker entries that a replay read again | +| `temporal_stream_replay_read_latency` | The time of one such read and its check | From 4ba8729db04f0902e1c4d4b61978700eabb256d8 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 23:20:42 -0700 Subject: [PATCH 34/59] Tested the Stream metrics and a shutdown with a close in flight. One Workflow reads, appends and closes, and the buffer of its Runtime has each metric with its count and Store. A shutdown returns after a close that the code didn't await. --- tests/nexus/test_stream_direct.py | 75 ++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 7685bfe50..a955150c3 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -39,7 +39,8 @@ StreamRef, StreamStatus, ) -from temporalio.testing import MemoryStreamStore +from temporalio.runtime import MetricBuffer, Runtime, TelemetryConfig +from temporalio.testing import MemoryStreamStore, WorkflowEnvironment from temporalio.worker import Replayer, Worker from tests.helpers import new_worker from tests.helpers.streams import DictStreamStore @@ -1700,6 +1701,22 @@ async def closed() -> bool: assert [entry.resolved for entry in entries] == [False] +async def test_shutdown_waits_for_a_close_not_awaited(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, CloseNotAwaitedWorkflow) as worker: + result = await stream_client.execute_workflow( + CloseNotAwaitedWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert result == "returned" + # A graceful shutdown returns only after the Store answered the close in flight, so + # the Stream is closed with no wait here. + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + + @activity.defn async def sleep_a_while() -> None: await asyncio.sleep(2) @@ -1751,3 +1768,59 @@ async def test_python_store_closes_through_its_activity(client: Client): history = await handle.fetch_history() assert _scheduled_activities(history) == ["__temporal_stream_store_call:dict"] assert _close_entries(history) == [] + + +# The metrics of the Direct calls. + + +@workflow.defn +class ReadAppendCloseWorkflow: + @workflow.run + async def run(self, source_id: str, out_id: str) -> list[str]: + source = workflow.get_stream(StreamRef(id=source_id), body_type=str) + out = workflow.get_stream(StreamRef(id=out_id), body_type=str) + got = [record.body async for record in source.iter()] + await out.append(*got) + await out.close() + return got + + +async def test_direct_calls_record_their_metrics(env: WorkflowEnvironment): + buffer = MetricBuffer(10000) + runtime = Runtime(telemetry=TelemetryConfig(metrics=buffer)) + client = await env.connect_client(runtime=runtime) + store = MemoryStreamStore(name="metered", runtime=runtime) + stream_client = _stream_client(client, store) + source = await _filled_stream(stream_client, ["a", "b", "c"]) + out = await _filled_stream(stream_client, [], close=False) + async with new_worker(stream_client, ReadAppendCloseWorkflow) as worker: + result = await stream_client.execute_workflow( + ReadAppendCloseWorkflow.run, + args=[source.id, out.id], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert result == ["a", "b", "c"] + updates = [ + update + for update in buffer.retrieve_updates() + if update.metric.name.startswith("temporal_stream_") + ] + print("\nStream metrics:", sorted({update.metric.name for update in updates})) + + def values(name: str) -> list[Any]: + return [update.value for update in updates if update.metric.name == name] + + assert sum(values("temporal_stream_read_total")) == 1 + assert sum(values("temporal_stream_read_records")) == 3 + assert sum(values("temporal_stream_append_total")) == 1 + assert sum(values("temporal_stream_close_total")) == 1 + assert len(values("temporal_stream_append_latency")) == 1 + assert len(values("temporal_stream_close_latency")) == 1 + assert values("temporal_stream_read_failed") == [] + assert values("temporal_stream_append_failed") == [] + assert values("temporal_stream_close_failed") == [] + for update in updates: + assert update.attributes["stream_store"] == "metered" + assert update.attributes["workflow_type"] == "ReadAppendCloseWorkflow" + assert update.attributes["task_queue"] == worker.task_queue From 1cd1d9c949076039a4bea01333a0ab8fecf17ee3 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 23:27:32 -0700 Subject: [PATCH 35/59] Pinned the nexgen fork commit that has the helper method in one PR. The fork has `@nexus.helper`, its `sync` form and `@nexus.support-type` in one pull request. The generated Stream package is the same with it. --- .github/workflows/ci.yml | 2 +- scripts/gen_nexus_system_api.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba53ed32b..8804392a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,7 +229,7 @@ jobs: - run: uv sync --all-extras - run: cargo install --locked nexgen --version 0.2.8 --features advanced --force # The Stream package needs `@nexus.helper` with `sync=true`, which only a nexgen fork has. - - run: cargo install --locked --git https://github.com/moetemp/nexgen --rev 42c7e43799f97c837346f1562db13493d9869858 --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen + - run: cargo install --locked --git https://github.com/moetemp/nexgen --rev e4b1f297e7a3fa8bc20fbf176fa2727c9e83a20f --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen - run: echo "NEX_GEN_STREAM_BIN=$RUNNER_TEMP/nexgen-stream/bin/nexgen" >> "$GITHUB_ENV" - run: poe build-develop - run: poe gen-protos diff --git a/scripts/gen_nexus_system_api.py b/scripts/gen_nexus_system_api.py index 065805135..4c725250a 100644 --- a/scripts/gen_nexus_system_api.py +++ b/scripts/gen_nexus_system_api.py @@ -30,7 +30,7 @@ # The Stream package needs a newer nexgen, which rejects the other WITs. One pin is enough # again once the api WITs and a nexgen release agree. The Stream WIT uses `@nexus.helper` with # `sync=true` and `@nexus.support-type`, which only the fork https://github.com/moetemp/nexgen -# has, at commit 42c7e437. So NEX_GEN_STREAM_BIN points at a build of that commit. +# has, at commit e4b1f297. So NEX_GEN_STREAM_BIN points at a build of that commit. NEX_GEN_STREAM_VERSION = "0.2.12" proto_files = [ gen_protos.api_proto_dir @@ -88,7 +88,7 @@ def nex_gen_stream_command() -> list[str]: bin_path = os.environ.get("NEX_GEN_STREAM_BIN") if not bin_path: raise RuntimeError( - "NEX_GEN_STREAM_BIN must point at a nexgen build of commit 42c7e437 " + "NEX_GEN_STREAM_BIN must point at a nexgen build of commit e4b1f297 " "of https://github.com/moetemp/nexgen" ) version = subprocess.check_output([bin_path, "--version"], text=True).strip() From 692263945d67eb19419e4af166828aefe44cf20c Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 23:52:17 -0700 Subject: [PATCH 36/59] Moved Core to the read-ahead option and the smaller Stream marker. The marker now holds each Stream ref once, so the generated stubs of `external_data` change. No Python code reads a marker entry's Stream ref. --- .../proto/external_data/external_data_pb2.py | 20 +++--- .../proto/external_data/external_data_pb2.pyi | 68 ++++++++++++------- temporalio/bridge/sdk-core | 2 +- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.py b/temporalio/bridge/proto/external_data/external_data_pb2.py index c2123054a..3e13e28c1 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.py +++ b/temporalio/bridge/proto/external_data/external_data_pb2.py @@ -27,7 +27,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a%temporal/sdk/core/common/common.proto"\xb2\x02\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x10\x61\x63tivation_index\x18\x08 \x01(\x04H\x00\x88\x01\x01\x42\x13\n\x11_activation_index"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xbd\x01\n\x10StreamMarkerData\x12\x35\n\x05reads\x18\x01 \x03(\x0b\x32&.coresdk.external_data.StreamReadEntry\x12\x39\n\x07\x61ppends\x18\x02 \x03(\x0b\x32(.coresdk.external_data.StreamAppendEntry\x12\x37\n\x06\x63loses\x18\x03 \x03(\x0b\x32\'.coresdk.external_data.StreamCloseEntry"\xcb\x02\n\x0fStreamReadEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12\x14\n\x0c\x61\x66ter_offset\x18\x04 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x05 \x01(\x0c\x12\r\n\x05\x63ount\x18\x06 \x01(\r\x12\x0e\n\x06\x64igest\x18\x07 \x01(\x0c\x12\x35\n\x07records\x18\x08 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x13\n\x0bpage_counts\x18\t \x03(\r\x12\x18\n\x10page_activations\x18\n \x03(\r\x12\x33\n\x03\x65nd\x18\x0b \x01(\x0b\x32&.coresdk.external_data.StreamReadEnded"\xa9\x01\n\x0fStreamReadEnded\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x05 \x01(\r"\xc5\x01\n\x11StreamAppendEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x14\n\x0c\x66irst_offset\x18\x03 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x04 \x01(\x0c\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\r"\xec\x01\n\x10StreamCloseEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12?\n\x0cintent_epoch\x18\x03 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch\x12\x10\n\x08resolved\x18\x04 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\rB2\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' + b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a%temporal/sdk/core/common/common.proto"\xb2\x02\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x10\x61\x63tivation_index\x18\x08 \x01(\x04H\x00\x88\x01\x01\x42\x13\n\x11_activation_index"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xf1\x01\n\x10StreamMarkerData\x12\x35\n\x05reads\x18\x01 \x03(\x0b\x32&.coresdk.external_data.StreamReadEntry\x12\x39\n\x07\x61ppends\x18\x02 \x03(\x0b\x32(.coresdk.external_data.StreamAppendEntry\x12\x37\n\x06\x63loses\x18\x03 \x03(\x0b\x32\'.coresdk.external_data.StreamCloseEntry\x12\x32\n\x07streams\x18\x04 \x03(\x0b\x32!.temporal.api.stream.v1.StreamRef"\xae\x02\n\x0fStreamReadEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x14\n\x0cstream_index\x18\x02 \x01(\r\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12\x14\n\x0c\x61\x66ter_offset\x18\x04 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x05 \x01(\x0c\x12\r\n\x05\x63ount\x18\x06 \x01(\r\x12\x0e\n\x06\x64igest\x18\x07 \x01(\x0c\x12\x35\n\x07records\x18\x08 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x13\n\x0bpage_counts\x18\t \x03(\r\x12\x18\n\x10page_activations\x18\n \x03(\r\x12\x33\n\x03\x65nd\x18\x0b \x01(\x0b\x32&.coresdk.external_data.StreamReadEnded"\xa9\x01\n\x0fStreamReadEnded\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x05 \x01(\r"\xa8\x01\n\x11StreamAppendEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x14\n\x0cstream_index\x18\x02 \x01(\r\x12\x14\n\x0c\x66irst_offset\x18\x03 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x04 \x01(\x0c\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\r"\xcf\x01\n\x10StreamCloseEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x14\n\x0cstream_index\x18\x02 \x01(\r\x12?\n\x0cintent_epoch\x18\x03 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch\x12\x10\n\x08resolved\x18\x04 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\rB2\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -42,13 +42,13 @@ _PATCHEDMARKERDATA._serialized_start = 568 _PATCHEDMARKERDATA._serialized_end = 619 _STREAMMARKERDATA._serialized_start = 622 - _STREAMMARKERDATA._serialized_end = 811 - _STREAMREADENTRY._serialized_start = 814 - _STREAMREADENTRY._serialized_end = 1145 - _STREAMREADENDED._serialized_start = 1148 - _STREAMREADENDED._serialized_end = 1317 - _STREAMAPPENDENTRY._serialized_start = 1320 - _STREAMAPPENDENTRY._serialized_end = 1517 - _STREAMCLOSEENTRY._serialized_start = 1520 - _STREAMCLOSEENTRY._serialized_end = 1756 + _STREAMMARKERDATA._serialized_end = 863 + _STREAMREADENTRY._serialized_start = 866 + _STREAMREADENTRY._serialized_end = 1168 + _STREAMREADENDED._serialized_start = 1171 + _STREAMREADENDED._serialized_end = 1340 + _STREAMAPPENDENTRY._serialized_start = 1343 + _STREAMAPPENDENTRY._serialized_end = 1511 + _STREAMCLOSEENTRY._serialized_start = 1514 + _STREAMCLOSEENTRY._serialized_end = 1721 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.pyi b/temporalio/bridge/proto/external_data/external_data_pb2.pyi index 840663e40..61b500977 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.pyi +++ b/temporalio/bridge/proto/external_data/external_data_pb2.pyi @@ -158,6 +158,7 @@ class StreamMarkerData(google.protobuf.message.Message): READS_FIELD_NUMBER: builtins.int APPENDS_FIELD_NUMBER: builtins.int CLOSES_FIELD_NUMBER: builtins.int + STREAMS_FIELD_NUMBER: builtins.int @property def reads( self, @@ -176,17 +177,38 @@ class StreamMarkerData(google.protobuf.message.Message): ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ global___StreamCloseEntry ]: ... + @property + def streams( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRef + ]: + """Each Stream of the entries, once. An entry gives the index of its Stream here, so that a task + with many results on one Stream holds its ref once. + """ + def __init__( self, *, reads: collections.abc.Iterable[global___StreamReadEntry] | None = ..., appends: collections.abc.Iterable[global___StreamAppendEntry] | None = ..., closes: collections.abc.Iterable[global___StreamCloseEntry] | None = ..., + streams: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRef + ] + | None = ..., ) -> None: ... def ClearField( self, field_name: typing.Literal[ - "appends", b"appends", "closes", b"closes", "reads", b"reads" + "appends", + b"appends", + "closes", + b"closes", + "reads", + b"reads", + "streams", + b"streams", ], ) -> None: ... @@ -199,7 +221,7 @@ class StreamReadEntry(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor SEQ_FIELD_NUMBER: builtins.int - STREAM_FIELD_NUMBER: builtins.int + STREAM_INDEX_FIELD_NUMBER: builtins.int FROM_OFFSET_FIELD_NUMBER: builtins.int AFTER_OFFSET_FIELD_NUMBER: builtins.int LAST_OFFSET_FIELD_NUMBER: builtins.int @@ -210,6 +232,8 @@ class StreamReadEntry(google.protobuf.message.Message): PAGE_ACTIVATIONS_FIELD_NUMBER: builtins.int END_FIELD_NUMBER: builtins.int seq: builtins.int + stream_index: builtins.int + """The index of the read's Stream in `StreamMarkerData.streams`.""" from_offset: builtins.bytes """The `from_offset` of the read, so a replay can check that the read is the same.""" after_offset: builtins.bytes @@ -221,8 +245,6 @@ class StreamReadEntry(google.protobuf.message.Message): digest: builtins.bytes """The digest of the Records of this task. Empty when `records` holds them.""" @property - def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... - @property def records( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ @@ -252,7 +274,7 @@ class StreamReadEntry(google.protobuf.message.Message): self, *, seq: builtins.int = ..., - stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + stream_index: builtins.int = ..., from_offset: builtins.bytes = ..., after_offset: builtins.bytes = ..., last_offset: builtins.bytes = ..., @@ -266,9 +288,7 @@ class StreamReadEntry(google.protobuf.message.Message): page_activations: collections.abc.Iterable[builtins.int] | None = ..., end: global___StreamReadEnded | None = ..., ) -> None: ... - def HasField( - self, field_name: typing.Literal["end", b"end", "stream", b"stream"] - ) -> builtins.bool: ... + def HasField(self, field_name: typing.Literal["end", b"end"]) -> builtins.bool: ... def ClearField( self, field_name: typing.Literal[ @@ -292,8 +312,8 @@ class StreamReadEntry(google.protobuf.message.Message): b"records", "seq", b"seq", - "stream", - b"stream", + "stream_index", + b"stream_index", ], ) -> None: ... @@ -352,32 +372,32 @@ class StreamAppendEntry(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor SEQ_FIELD_NUMBER: builtins.int - STREAM_FIELD_NUMBER: builtins.int + STREAM_INDEX_FIELD_NUMBER: builtins.int FIRST_OFFSET_FIELD_NUMBER: builtins.int LAST_OFFSET_FIELD_NUMBER: builtins.int FAILURE_FIELD_NUMBER: builtins.int ACTIVATION_FIELD_NUMBER: builtins.int seq: builtins.int + stream_index: builtins.int + """The index of the append's Stream in `StreamMarkerData.streams`.""" first_offset: builtins.bytes last_offset: builtins.bytes activation: builtins.int """The activation in the task that got the result.""" @property - def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... - @property def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... def __init__( self, *, seq: builtins.int = ..., - stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + stream_index: builtins.int = ..., first_offset: builtins.bytes = ..., last_offset: builtins.bytes = ..., failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., activation: builtins.int = ..., ) -> None: ... def HasField( - self, field_name: typing.Literal["failure", b"failure", "stream", b"stream"] + self, field_name: typing.Literal["failure", b"failure"] ) -> builtins.bool: ... def ClearField( self, @@ -392,8 +412,8 @@ class StreamAppendEntry(google.protobuf.message.Message): b"last_offset", "seq", b"seq", - "stream", - b"stream", + "stream_index", + b"stream_index", ], ) -> None: ... @@ -408,19 +428,19 @@ class StreamCloseEntry(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor SEQ_FIELD_NUMBER: builtins.int - STREAM_FIELD_NUMBER: builtins.int + STREAM_INDEX_FIELD_NUMBER: builtins.int INTENT_EPOCH_FIELD_NUMBER: builtins.int RESOLVED_FIELD_NUMBER: builtins.int FAILURE_FIELD_NUMBER: builtins.int ACTIVATION_FIELD_NUMBER: builtins.int seq: builtins.int + stream_index: builtins.int + """The index of the closed Stream in `StreamMarkerData.streams`.""" resolved: builtins.bool """True on the result. `failure` is unset when the Stream is closed.""" activation: builtins.int """The activation in the task that got the result.""" @property - def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... - @property def intent_epoch(self) -> temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch: """Set on the intent, the epoch that Core closes with after the server took the task.""" @@ -430,7 +450,7 @@ class StreamCloseEntry(google.protobuf.message.Message): self, *, seq: builtins.int = ..., - stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + stream_index: builtins.int = ..., intent_epoch: temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch | None = ..., resolved: builtins.bool = ..., @@ -440,7 +460,7 @@ class StreamCloseEntry(google.protobuf.message.Message): def HasField( self, field_name: typing.Literal[ - "failure", b"failure", "intent_epoch", b"intent_epoch", "stream", b"stream" + "failure", b"failure", "intent_epoch", b"intent_epoch" ], ) -> builtins.bool: ... def ClearField( @@ -456,8 +476,8 @@ class StreamCloseEntry(google.protobuf.message.Message): b"resolved", "seq", b"seq", - "stream", - b"stream", + "stream_index", + b"stream_index", ], ) -> None: ... diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 9d7e25131..2c7f87339 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 9d7e25131ec70c06cc6921f9c42896df4304ca0d +Subproject commit 2c7f87339a9ed7230e00cfe40247ab68285605de From 947a8fe6d140c3a23656cb0c0f0eb3944a55c0e8 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 23:55:02 -0700 Subject: [PATCH 37/59] Added `StreamConfig.workflow_read_ahead_bytes` for Workflow reads. A read stops asking the Store while that many bytes of its Records wait for Workflow code. The Worker and the Replayer give the bound to Core through the bridge. --- temporalio/bridge/src/worker.rs | 7 +++++++ temporalio/bridge/worker.py | 2 ++ temporalio/client/_stream_store.py | 18 +++++++++++++++++- temporalio/worker/_replayer.py | 9 ++++++++- temporalio/worker/_stream_store.py | 5 +++++ temporalio/worker/_worker.py | 7 ++++++- 6 files changed, 45 insertions(+), 3 deletions(-) diff --git a/temporalio/bridge/src/worker.rs b/temporalio/bridge/src/worker.rs index ad7f7c611..9b34a203b 100644 --- a/temporalio/bridge/src/worker.rs +++ b/temporalio/bridge/src/worker.rs @@ -69,6 +69,9 @@ pub struct WorkerConfig { /// Stream of Workflow code through them. #[cfg(feature = "streams")] stream_runtimes: Vec<(String, Py)>, + /// The most bytes of one read's Records that wait for Workflow code. `None` keeps Core's + /// default. + stream_read_ahead_bytes: Option, } #[derive(FromPyObject)] @@ -731,6 +734,10 @@ fn convert_worker_config( ); #[cfg(not(feature = "streams"))] let stream_runtimes = temporalio_sdk_core::StreamRuntimes::default(); + let stream_runtimes = match conf.stream_read_ahead_bytes { + Some(bytes) => stream_runtimes.with_read_ahead_bytes(bytes), + None => stream_runtimes, + }; temporalio_sdk_core::WorkerConfig::builder() .namespace(conf.namespace) .task_queue(conf.task_queue) diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index 616e75690..7400642bf 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -63,6 +63,8 @@ class WorkerConfig: disable_payload_error_limit: bool stream_runtimes: Sequence[tuple[str, Any]] = () """The bridge's object of each Store's Stream runtime, by Store name.""" + stream_read_ahead_bytes: int | None = None + """The most bytes of one read's Records that wait for Workflow code, or Core's default.""" @dataclass diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index ca9546eb8..610c56295 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -556,6 +556,16 @@ class StreamConfig: this setting, so a Workflow that is running replays after a change of it. """ + workflow_read_ahead_bytes: int | None = None + """The most bytes of one read's Records that wait in the Worker for Workflow code. + + A read of Workflow code with ``iter`` stops asking the Store while this many bytes of + its Records wait in the Worker's memory for Workflow code to take them. The read goes + on when Workflow code takes Records. The first page of a read always comes. ``None`` + takes Core's default of 4 MiB. Set a larger value for large Records, and a smaller one + for a Worker with little memory, since each open read can hold this much. + """ + _store_map: dict[str, StreamStore] = dataclasses.field( init=False, repr=False, compare=False ) @@ -565,8 +575,14 @@ def __post_init__(self) -> None: """Check the Stores and build the name-keyed map of them. Raises :py:class:`temporalio.exceptions.StreamConfigError` if there is no - Store, or if two Stores share a name. + Store, or if two Stores share a name, and ``ValueError`` if + :py:attr:`workflow_read_ahead_bytes` is below 1. """ + if ( + self.workflow_read_ahead_bytes is not None + and self.workflow_read_ahead_bytes < 1 + ): + raise ValueError("workflow_read_ahead_bytes must be at least 1") if not self.stores: raise StreamConfigError("StreamConfig.stores must have at least one Store") store_map: dict[str, StreamStore] = {} diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index 1078b4ccf..cf04a2cea 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -22,7 +22,11 @@ from ..common import HeaderCodecBehavior from ._interceptor import Interceptor -from ._stream_store import _bridge_stream_runtimes, _WorkflowStreamStore +from ._stream_store import ( + _bridge_read_ahead_bytes, + _bridge_stream_runtimes, + _WorkflowStreamStore, +) from ._worker import load_default_build_id from ._workflow import _WorkflowWorker from ._workflow_instance import ( @@ -380,6 +384,9 @@ def on_eviction_hook( stream_runtimes=_bridge_stream_runtimes( self._config.get("stream_config") ), + stream_read_ahead_bytes=_bridge_read_ahead_bytes( + self._config.get("stream_config") + ), ), ) bridge_worker_scope = bridge_worker diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index 836f0506e..5484d2673 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -219,6 +219,11 @@ def _read_command( # pyright: ignore[reportUnusedFunction] return start +def _bridge_read_ahead_bytes(config: StreamConfig | None) -> int | None: # pyright: ignore[reportUnusedFunction] + """Give Core the read-ahead bound of the config, or ``None`` for Core's default.""" + return config.workflow_read_ahead_bytes if config is not None else None + + def _bridge_stream_runtimes(config: StreamConfig | None) -> list[tuple[str, Any]]: # pyright: ignore[reportUnusedFunction] """Give Core the bridge's object of each Store's Stream runtime, by Store name.""" if config is None: diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index 8d7b7ee55..709ac1d2b 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -34,7 +34,11 @@ from ._interceptor import Interceptor from ._nexus import _NexusWorker from ._plugin import Plugin -from ._stream_store import _bridge_stream_runtimes, _WorkflowStreamStore +from ._stream_store import ( + _bridge_read_ahead_bytes, + _bridge_stream_runtimes, + _WorkflowStreamStore, +) from ._tuning import WorkerTuner from ._workflow import ( _DEFAULT_WORKFLOW_TASK_EXTERNAL_STORAGE_CONCURRENCY, @@ -729,6 +733,7 @@ def check_activity(activity: str): stream_runtimes=_bridge_stream_runtimes( stream_config if self._workflow_worker is not None else None ), + stream_read_ahead_bytes=_bridge_read_ahead_bytes(stream_config), ), ) From 9902b3ba5599b56b89e51f0c1bfae7777346e3be Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 23:54:55 -0700 Subject: [PATCH 38/59] Showed a decision in Workflow code as the example of `iter`. A copy of each Record to another Stream is work that an Activity could do itself. The example now starts each tool as its call arrives, and the docs tell of the read-ahead bound. --- changelog/added/patient-river-heron.md | 1 + temporalio/contrib/streams/README.md | 25 +++++++++++++++++++++---- temporalio/nexus/system/_support.py | 16 ++++++++++++++-- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md index 238a196a3..4d1d6842c 100644 --- a/changelog/added/patient-river-heron.md +++ b/changelog/added/patient-river-heron.md @@ -2,3 +2,4 @@ Add `iter` to the Workflow `Stream`, so Workflow code reads each Record of a Str A Workflow's append on the Redis Store or `MemoryStreamStore` now runs in Core inside the Workflow Task, with no Local Activity. It writes as the Workflow, with its Workflow Task as the epoch. A Workflow's close on the Redis Store or `MemoryStreamStore` now runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close: a graceful shutdown of the Worker waits at most 10 s for a close that the code doesn't await, and a Worker that dies when the Workflow ends loses it. An append and a close of Workflow code always carry the Workflow Task as their epoch. Core records metrics for the Direct calls of a Stream, such as `temporal_stream_read_total`, `temporal_stream_read_records` and `temporal_stream_close_latency`, each with the attribute `stream_store`. +Add `StreamConfig.workflow_read_ahead_bytes`, the most bytes of one read's Records that wait in the Worker for Workflow code, 4 MiB by default. diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 097690fa2..9c4e3e445 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -51,11 +51,24 @@ reads, appends and closes a Stream itself, with no Activity. A read is `iter` of `Stream`. It gives each Record as it arrives and ends at a stop. By default a read has no bound, so it reads until the Stream is closed and the read gave its last Record. +Here the Workflow reads the events of a model's answer while the model Activity still runs, +and starts each tool as its call arrives. + ```python -tokens = workflow.get_stream(tokens_ref, body_type=Token) -shown = workflow.get_stream(shown_ref, body_type=str) -async for record in tokens.iter(): - await shown.append(record.body.text) +events = workflow.get_stream(events_ref, body_type=ModelEvent) +status = workflow.get_stream(status_ref, body_type=str) +tools = [] +async for record in events.iter(): + event = record.body + if event.kind == "tool_call": + # The tool starts now, before the model ends its answer. + tools.append( + workflow.start_activity( + run_tool, event.tool_call, start_to_close_timeout=timedelta(minutes=1) + ) + ) + await status.append(f"Running {event.tool_call.name}") +results = await asyncio.gather(*tools) ``` `iter` takes `from_offset` and four bounds, `timeout`, `idle_timeout`, `max_records` and @@ -99,6 +112,10 @@ of Records. It has three other costs. 256 KiB. - A replay on a Worker with no cache reads the Store again on every Workflow Task. +While Workflow code is busy, a read holds at most 4 MiB of its Records in the Worker, then +waits until the code takes them. `workflow_read_ahead_bytes` of `StreamConfig` changes that +bound, for large Records or a Worker with little memory. + An append and a close of Workflow code always write as the Workflow, with the Workflow Task as their epoch. An append runs inside its Workflow Task and records a marker with no body. diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index 5025a088d..d6290dbb2 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -389,8 +389,20 @@ def iter_stream( .. code-block:: python - async for record in tokens.iter(): - await shown.append(record.body.text) + tools = [] + async for record in events.iter(): + event = record.body + if event.kind == "tool_call": + # The tool starts now, before the model ends its answer. + tools.append( + workflow.start_activity( + run_tool, + event.tool_call, + start_to_close_timeout=timedelta(minutes=1), + ) + ) + await status.append(f"Running {event.tool_call.name}") + results = await asyncio.gather(*tools) A cap, a ``timeout`` or an ``idle_timeout`` ends a read before the Stream is done. A loop of reads from ``last_offset`` then goes on where the last read stopped. From 0f9f0858b937c48888c14c9eefaf22ebe7ac510d Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 23:54:55 -0700 Subject: [PATCH 39/59] Tested the tool-call example of `iter` and the read-ahead bound. The first tool starts before the model Activity completes. A bound below 1 raises, and a bound of 1 byte still gives every Record in order. --- tests/nexus/test_stream_direct.py | 125 ++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index a955150c3..c46bf5c76 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -1824,3 +1824,128 @@ def values(name: str) -> list[Any]: assert update.attributes["stream_store"] == "metered" assert update.attributes["workflow_type"] == "ReadAppendCloseWorkflow" assert update.attributes["task_queue"] == worker.task_queue + + +# The example of `iter`: a Workflow starts each tool as its call arrives. + + +@dataclass +class ToolCall: + name: str + argument: str + + +@dataclass +class ModelEvent: + kind: str + text: str | None = None + tool_call: ToolCall | None = None + + +_MODEL_EVENTS = [ + ModelEvent(kind="text", text="Let me look."), + ModelEvent(kind="tool_call", tool_call=ToolCall(name="weather", argument="Oslo")), + ModelEvent(kind="text", text="And the time."), + ModelEvent(kind="tool_call", tool_call=ToolCall(name="clock", argument="Oslo")), + ModelEvent(kind="text", text="Done."), +] + + +@activity.defn +async def model_answer(events_id: str) -> None: + events = activity.client().get_stream(StreamRef(id=events_id), body_type=ModelEvent) + for event in _MODEL_EVENTS: + await events.append(event) + # The model takes a while between its events, so a tool can start before it ends. + await asyncio.sleep(0.5) + await events.close() + + +@activity.defn +async def run_tool(call: ToolCall) -> str: + return f"{call.name}({call.argument})" + + +@workflow.defn +class ToolCallsWorkflow: + @workflow.run + async def run(self, events_id: str, status_id: str) -> list[str]: + model = workflow.start_activity( + model_answer, events_id, start_to_close_timeout=timedelta(seconds=30) + ) + events = workflow.get_stream(StreamRef(id=events_id), body_type=ModelEvent) + status = workflow.get_stream(StreamRef(id=status_id), body_type=str) + tools = [] + async for record in events.iter(): + event = record.body + if event.kind == "tool_call": + assert event.tool_call is not None + # The tool starts now, before the model ends its answer. + tools.append( + workflow.start_activity( + run_tool, + event.tool_call, + start_to_close_timeout=timedelta(minutes=1), + ) + ) + await status.append(f"Running {event.tool_call.name}") + results = await asyncio.gather(*tools) + await model + return list(results) + + +async def test_workflow_starts_each_tool_as_its_call_arrives(client: Client): + stream_client = _stream_client(client) + events = await _filled_stream(stream_client, [], close=False) + status = await _filled_stream(stream_client, [], close=False) + handle, results = await _run( + stream_client, + ToolCallsWorkflow, + events.id, + status.id, + activities=[model_answer, run_tool], + ) + assert results == ["weather(Oslo)", "clock(Oslo)"] + assert [record.body for record in await _read_all(stream_client, status)] == [ + "Running weather", + "Running clock", + ] + # The first tool was scheduled while the model Activity still ran. + history = await handle.fetch_history() + scheduled: dict[int, str] = {} + first_tool = None + model_completed = None + for event in history.events: + if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED: + name = event.activity_task_scheduled_event_attributes.activity_type.name + scheduled[event.event_id] = name + if name == "run_tool" and first_tool is None: + first_tool = event.event_id + elif event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_COMPLETED: + attributes = event.activity_task_completed_event_attributes + if scheduled[attributes.scheduled_event_id] == "model_answer": + model_completed = event.event_id + assert first_tool is not None and model_completed is not None + assert first_tool < model_completed + + +# The read-ahead bound. + + +def test_read_ahead_bytes_below_one_raises(): + for bytes_ in (0, -1): + with pytest.raises(ValueError, match="workflow_read_ahead_bytes"): + StreamConfig(stores=[MemoryStreamStore()], workflow_read_ahead_bytes=bytes_) + + +async def test_read_with_a_read_ahead_of_one_byte_gives_every_record(client: Client): + config = client.config() + config["stream_config"] = StreamConfig( + stores=[MemoryStreamStore()], workflow_read_ahead_bytes=1 + ) + stream_client = Client(**config) + bodies = [f"record-{index}" for index in range(20)] + ref = await _filled_stream(stream_client, bodies) + _, result = await _run(stream_client, ReadOnceWorkflow, ReadArgs(ref.id)) + assert result.bodies == bodies + assert (result.stop, result.done) == ("Done", True) From ded2c2997d14574b449854da100f5963a8456c96 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 01:05:04 -0700 Subject: [PATCH 40/59] Moved Core to the API's `WorkflowProducer` comment. The stub of the Stream messages has the new comment, which says who sets the epoch. --- temporalio/api/stream/v1/message_pb2.pyi | 8 +++++--- temporalio/bridge/sdk-core | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/temporalio/api/stream/v1/message_pb2.pyi b/temporalio/api/stream/v1/message_pb2.pyi index 1d0103602..1e42e74a2 100644 --- a/temporalio/api/stream/v1/message_pb2.pyi +++ b/temporalio/api/stream/v1/message_pb2.pyi @@ -167,8 +167,8 @@ global___ActivityEpoch = ActivityEpoch @typing.final class WorkflowProducer(google.protobuf.message.Message): """Experimental. Might change in the future. - A Workflow run Producer. On the server's Streams, a Workflow writes as this Producer in its - Workflow Task. On a Store, Workflow code writes through a Local Activity, so its writes carry a + A Workflow run Producer. Workflow code that writes in its Workflow Task writes as this + Producer, with that task as the epoch. A write that it makes through a Local Activity has a Local Activity Producer. """ @@ -180,7 +180,9 @@ class WorkflowProducer(google.protobuf.message.Message): """Required. The run id of the Workflow run, not of its chain. Each run is its own Producer.""" @property def epoch(self) -> global___WorkflowTaskEpoch: - """The Workflow Task of the write.""" + """The Workflow Task of the write. Workflow code leaves it empty, and the SDK sets it before + the write goes to the Store. + """ def __init__( self, diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 2c7f87339..3ecf9e1e9 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 2c7f87339a9ed7230e00cfe40247ab68285605de +Subproject commit 3ecf9e1e913f5384ab73d196945f831e533c23cd From f1a36385c270debf0848e7762d0b129e06ed97fd Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 13:37:49 -0700 Subject: [PATCH 41/59] Moved Core to its review fixes of the Direct Stream calls. Core adds a batch window and an idle default of 1 minute to a read, and can fail the Workflow when a replay lost its Records. Only comments of the generated stubs change. --- .../proto/workflow_commands/workflow_commands_pb2.pyi | 10 ++++++---- temporalio/bridge/sdk-core | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi index 8e7292f5f..233572f0a 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi @@ -2001,14 +2001,16 @@ class StartStreamRead(google.protobuf.message.Message): def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... @property def timeout(self) -> google.protobuf.duration_pb2.Duration: - """The longest time the read may take. Unset means no bound. A read that gave no Record yet - looks at the Store once before its timeout ends it. + """The longest time the read may take. Unset means no bound. A set value of zero or less fails + the workflow task. A read that gave no Record yet looks at the Store once before its timeout + ends it. """ @property def idle_timeout(self) -> google.protobuf.duration_pb2.Duration: - """The read ends when no Record comes for this long. Unset means no bound. Zero gives the - Records that are in the Stream, and ends with no wait. + """The read ends when no Record comes for this long. Unset means 1 minute. Zero gives the + Records that are in the Stream, and ends with no wait. A read that must wait longer sets a + longer value. """ def __init__( diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 3ecf9e1e9..2c1c85fe2 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 3ecf9e1e913f5384ab73d196945f831e533c23cd +Subproject commit 2c1c85fe2ff0580fbedd5d6a7f14d738372fcff4 From a71f07746bf7a118c0470d95362313ac63145749 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 13:43:19 -0700 Subject: [PATCH 42/59] Added a read batch window and a Workflow failure for lost Records. `StreamConfig.workflow_read_batch_window` bounds how often a read activates a Workflow. `StreamReplayError` in the failure types fails the Workflow when a replay lost its Records. --- temporalio/bridge/src/worker.rs | 51 ++++++++++++++++++++---------- temporalio/bridge/worker.py | 9 ++++++ temporalio/client/_stream_store.py | 19 ++++++++++- temporalio/exceptions.py | 19 +++++++++++ temporalio/worker/_replayer.py | 8 +++++ temporalio/worker/_stream_store.py | 9 ++++++ temporalio/worker/_worker.py | 11 +++++++ temporalio/worker/_workflow.py | 16 ++++++++++ 8 files changed, 124 insertions(+), 18 deletions(-) diff --git a/temporalio/bridge/src/worker.rs b/temporalio/bridge/src/worker.rs index 9b34a203b..a2ec86a24 100644 --- a/temporalio/bridge/src/worker.rs +++ b/temporalio/bridge/src/worker.rs @@ -5,7 +5,7 @@ use prost::Message; use pyo3::exceptions::{PyException, PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyTuple}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::sync::{Arc, OnceLock}; use std::time::Duration; @@ -72,6 +72,12 @@ pub struct WorkerConfig { /// The most bytes of one read's Records that wait for Workflow code. `None` keeps Core's /// default. stream_read_ahead_bytes: Option, + /// The least time between two batches of one read. `None` keeps Core's default. + stream_read_batch_window_millis: Option, + /// Whether a replay that lost the Records of a read fails the Workflow, for every type or + /// for the types listed. + stream_replay_as_workflow_fail: bool, + stream_replay_as_workflow_fail_for_types: HashSet, } #[derive(FromPyObject)] @@ -738,6 +744,31 @@ fn convert_worker_config( Some(bytes) => stream_runtimes.with_read_ahead_bytes(bytes), None => stream_runtimes, }; + let stream_runtimes = match conf.stream_read_batch_window_millis { + Some(millis) => stream_runtimes.with_read_batch_window(Duration::from_millis(millis)), + None => stream_runtimes, + }; + let mut workflow_failure_errors = HashSet::new(); + if conf.nondeterminism_as_workflow_fail { + workflow_failure_errors.insert(WorkflowErrorType::Nondeterminism); + } + if conf.stream_replay_as_workflow_fail { + workflow_failure_errors.insert(WorkflowErrorType::StreamReplay); + } + let mut workflow_types_to_failure_errors: HashMap> = + HashMap::new(); + for workflow_type in conf.nondeterminism_as_workflow_fail_for_types { + workflow_types_to_failure_errors + .entry(workflow_type) + .or_default() + .insert(WorkflowErrorType::Nondeterminism); + } + for workflow_type in conf.stream_replay_as_workflow_fail_for_types { + workflow_types_to_failure_errors + .entry(workflow_type) + .or_default() + .insert(WorkflowErrorType::StreamReplay); + } temporalio_sdk_core::WorkerConfig::builder() .namespace(conf.namespace) .task_queue(conf.task_queue) @@ -767,22 +798,8 @@ fn convert_worker_config( // auto-cancel-activity behavior of shutdown will not occur, so we // always set it even if 0. .graceful_shutdown_period(Duration::from_millis(conf.graceful_shutdown_period_millis)) - .workflow_failure_errors(if conf.nondeterminism_as_workflow_fail { - HashSet::from([WorkflowErrorType::Nondeterminism]) - } else { - HashSet::new() - }) - .workflow_types_to_failure_errors( - conf.nondeterminism_as_workflow_fail_for_types - .iter() - .map(|s| { - ( - s.to_owned(), - HashSet::from([WorkflowErrorType::Nondeterminism]), - ) - }) - .collect(), - ) + .workflow_failure_errors(workflow_failure_errors) + .workflow_types_to_failure_errors(workflow_types_to_failure_errors) .maybe_nexus_task_poller_behavior(conf.nexus_task_poller_behavior.map(Into::into)) .plugins( conf.plugins diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index 7400642bf..a03b6f224 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -5,6 +5,7 @@ from __future__ import annotations +import dataclasses from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass from typing import ( @@ -65,6 +66,14 @@ class WorkerConfig: """The bridge's object of each Store's Stream runtime, by Store name.""" stream_read_ahead_bytes: int | None = None """The most bytes of one read's Records that wait for Workflow code, or Core's default.""" + stream_read_batch_window_millis: int | None = None + """The least time between two batches of one read, or ``None`` for Core's default.""" + stream_replay_as_workflow_fail: bool = False + """Whether a replay that lost the Records of a read fails the Workflow.""" + stream_replay_as_workflow_fail_for_types: set[str] = dataclasses.field( + default_factory=set + ) + """The Workflow types for which such a replay fails the Workflow.""" @dataclass diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index 610c56295..aabd41d9a 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -566,6 +566,17 @@ class StreamConfig: for a Worker with little memory, since each open read can hold this much. """ + workflow_read_batch_window: timedelta | None = None + """The least time between two batches that one read gives to Workflow code. + + A read of Workflow code with ``iter`` gives its first batch immediately. After that it + gives Workflow code a batch at most once in this window, with every Record that came + in the window. So a busy Stream doesn't activate the Workflow for each Record. A Record + waits at most one window. ``None`` takes Core's default of 50 ms, and zero is no + window. Set a longer window when a busy Stream activates the Workflow too often, and + zero when each Record must reach Workflow code as soon as it comes. + """ + _store_map: dict[str, StreamStore] = dataclasses.field( init=False, repr=False, compare=False ) @@ -576,13 +587,19 @@ def __post_init__(self) -> None: Raises :py:class:`temporalio.exceptions.StreamConfigError` if there is no Store, or if two Stores share a name, and ``ValueError`` if - :py:attr:`workflow_read_ahead_bytes` is below 1. + :py:attr:`workflow_read_ahead_bytes` is below 1 or + :py:attr:`workflow_read_batch_window` is negative. """ if ( self.workflow_read_ahead_bytes is not None and self.workflow_read_ahead_bytes < 1 ): raise ValueError("workflow_read_ahead_bytes must be at least 1") + if ( + self.workflow_read_batch_window is not None + and self.workflow_read_batch_window < timedelta() + ): + raise ValueError("workflow_read_batch_window must not be negative") if not self.stores: raise StreamConfigError("StreamConfig.stores must have at least one Store") store_map: dict[str, StreamStore] = {} diff --git a/temporalio/exceptions.py b/temporalio/exceptions.py index 526c2eb47..5d2380cb4 100644 --- a/temporalio/exceptions.py +++ b/temporalio/exceptions.py @@ -316,6 +316,25 @@ class StreamConfigError(RuntimeError): """ +class StreamReplayError(TemporalError): + """A replay could not read the Records of a read of Workflow code from its Store again. + + A read under the default :py:attr:`temporalio.client.StreamReplaySource.STORE` keeps + no Record in History, so a replay reads the Records from the Store again. When the + Store doesn't have them, for example after the Stream's Retention, or they differ from + what History says, the replay can't go on. + + By default that fails only the Workflow Task, so a later attempt works once the + Records are back. With this class in ``workflow_failure_exception_types`` of the + Worker, or in ``failure_exception_types`` of ``@workflow.defn``, the Workflow fails + instead. Its failure is an :py:class:`ApplicationError` with the ``type`` + ``StreamReplayError``, and Core raises no instance of this class. + + .. warning:: + This API is experimental and subject to change. + """ + + class CancelledError(FailureError): """Error raised on workflow/activity cancellation.""" diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index cf04a2cea..b60948103 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -24,6 +24,7 @@ from ._interceptor import Interceptor from ._stream_store import ( _bridge_read_ahead_bytes, + _bridge_read_batch_window_millis, _bridge_stream_runtimes, _WorkflowStreamStore, ) @@ -332,6 +333,10 @@ def on_eviction_hook( # per workflow type nondeterminism_as_workflow_fail=workflow_worker.nondeterminism_as_workflow_fail(), nondeterminism_as_workflow_fail_for_types=workflow_worker.nondeterminism_as_workflow_fail_for_types(), + stream_replay_as_workflow_fail=workflow_worker.stream_replay_as_workflow_fail(), + stream_replay_as_workflow_fail_for_types=( + workflow_worker.stream_replay_as_workflow_fail_for_types() + ), # All values below are ignored but required by Core max_cached_workflows=2, tuner=temporalio.bridge.worker.TunerHolder( @@ -387,6 +392,9 @@ def on_eviction_hook( stream_read_ahead_bytes=_bridge_read_ahead_bytes( self._config.get("stream_config") ), + stream_read_batch_window_millis=_bridge_read_batch_window_millis( + self._config.get("stream_config") + ), ), ) bridge_worker_scope = bridge_worker diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index 5484d2673..798409d8b 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -224,6 +224,15 @@ def _bridge_read_ahead_bytes(config: StreamConfig | None) -> int | None: # pyri return config.workflow_read_ahead_bytes if config is not None else None +def _bridge_read_batch_window_millis( # pyright: ignore[reportUnusedFunction] + config: StreamConfig | None, +) -> int | None: + """Give Core the batch window of the config in milliseconds, or ``None`` for Core's default.""" + if config is None or config.workflow_read_batch_window is None: + return None + return round(config.workflow_read_batch_window.total_seconds() * 1000) + + def _bridge_stream_runtimes(config: StreamConfig | None) -> list[tuple[str, Any]]: # pyright: ignore[reportUnusedFunction] """Give Core the bridge's object of each Store's Stream runtime, by Store name.""" if config is None: diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index 709ac1d2b..78ced650c 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -36,6 +36,7 @@ from ._plugin import Plugin from ._stream_store import ( _bridge_read_ahead_bytes, + _bridge_read_batch_window_millis, _bridge_stream_runtimes, _WorkflowStreamStore, ) @@ -713,6 +714,13 @@ def check_activity(activity: str): if self._workflow_worker else set() ), + stream_replay_as_workflow_fail=self._workflow_worker is not None + and self._workflow_worker.stream_replay_as_workflow_fail(), + stream_replay_as_workflow_fail_for_types=( + self._workflow_worker.stream_replay_as_workflow_fail_for_types() + if self._workflow_worker + else set() + ), versioning_strategy=versioning_strategy, workflow_task_poller_behavior=_poller_behavior_to_bridge( workflow_task_poller_behavior @@ -734,6 +742,9 @@ def check_activity(activity: str): stream_config if self._workflow_worker is not None else None ), stream_read_ahead_bytes=_bridge_read_ahead_bytes(stream_config), + stream_read_batch_window_millis=_bridge_read_batch_window_millis( + stream_config + ), ), ) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 3e20e6adb..74e40dfd1 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -835,6 +835,12 @@ def nondeterminism_as_workflow_fail(self) -> bool: for typ in self._workflow_failure_exception_types ) + def stream_replay_as_workflow_fail(self) -> bool: + return any( + issubclass(temporalio.exceptions.StreamReplayError, typ) + for typ in self._workflow_failure_exception_types + ) + def _set_default_workflow_logic_flag( self, flag: _WorkflowLogicFlag, *, enabled: bool ) -> None: @@ -853,6 +859,16 @@ def nondeterminism_as_workflow_fail_for_types(self) -> set[str]: ) } + def stream_replay_as_workflow_fail_for_types(self) -> set[str]: + return { + k + for k, v in self._workflows.items() + if any( + issubclass(temporalio.exceptions.StreamReplayError, typ) + for typ in v.failure_exception_types + ) + } + class _DeadlockError(Exception): """Exception class for deadlocks. Contains functionality to swap the default traceback for another.""" From 05944151e6be1b8593a974fc53054a200dba878a Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 13:43:19 -0700 Subject: [PATCH 43/59] Documented the idle default of a read and Core's review fixes. A read with no `idle_timeout` now ends after 1 minute, so code on a quiet Stream checks `done`. The docs also say that an append is at-least-once, and give the 1 MiB marker rule. --- changelog/added/patient-river-heron.md | 8 ++++--- temporalio/contrib/streams/README.md | 31 +++++++++++++++++--------- temporalio/nexus/system/_support.py | 22 ++++++++++-------- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md index 4d1d6842c..0841527d4 100644 --- a/changelog/added/patient-river-heron.md +++ b/changelog/added/patient-river-heron.md @@ -1,5 +1,7 @@ -Add `iter` to the Workflow `Stream`, so Workflow code reads each Record of a Stream as it arrives. By default a read has no bound and reads until the Stream is closed. A read that waits on a quiet Stream holds its Workflow Task open, with one Workflow Task about every 8 s, and an `idle_timeout` ends it with `Idle`. Core reads the Store inside the Workflow Task, and a read costs one marker in History for each Workflow Task that it spans. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). +Add `iter` to the Workflow `Stream`, so Workflow code reads each Record of a Stream as it arrives. By default a read ends with `Idle` after 1 minute with no new Record, so code that reads a Stream that can be quiet for longer checks `done` after the loop or sets a longer `idle_timeout`. A read that waits on a quiet Stream holds its Workflow Task open, with one Workflow Task about every 8 s. Core reads the Store inside the Workflow Task, and a read costs one marker in History for each Workflow Task that it spans. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). With `HISTORY`, one Workflow Task's marker holds at most 1 MiB of Records, and the read goes on in the next task. A Workflow's append on the Redis Store or `MemoryStreamStore` now runs in Core inside the Workflow Task, with no Local Activity. It writes as the Workflow, with its Workflow Task as the epoch. -A Workflow's close on the Redis Store or `MemoryStreamStore` now runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close: a graceful shutdown of the Worker waits at most 10 s for a close that the code doesn't await, and a Worker that dies when the Workflow ends loses it. An append and a close of Workflow code always carry the Workflow Task as their epoch. -Core records metrics for the Direct calls of a Stream, such as `temporal_stream_read_total`, `temporal_stream_read_records` and `temporal_stream_close_latency`, each with the attribute `stream_store`. +A Workflow's close on the Redis Store or `MemoryStreamStore` now runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close: a graceful shutdown of the Worker waits at most 10 s for a close that the code doesn't await, and a Worker that dies when the Workflow ends loses it. An append and a close of Workflow code always carry the Workflow Task as their epoch. An append is at-least-once: after a Worker crash or an eviction, the rerun of its task can write a second copy with the epoch of the new task. +Core records metrics for the Direct calls of a Stream, such as `temporal_stream_read_total`, `temporal_stream_read_records` and `temporal_stream_close_latency`, each with the attribute `stream_store`, and `temporal_stream_close_abandoned` for a close that a graceful shutdown gave up. Add `StreamConfig.workflow_read_ahead_bytes`, the most bytes of one read's Records that wait in the Worker for Workflow code, 4 MiB by default. +Add `StreamConfig.workflow_read_batch_window`, the least time between two batches that one read gives to Workflow code, 50 ms by default. +Add `temporalio.exceptions.StreamReplayError`. With it in `workflow_failure_exception_types`, a replay that can't read the Records of a read from the Store again fails the Workflow, and not only the Workflow Task. diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 9c4e3e445..39da835d9 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -48,8 +48,8 @@ replacement of the first attempt's Records. The helpers run on the Streams API, so they follow how Workflow code reads, appends and closes. On a Store with a Rust runtime, such as the Redis Store or `MemoryStreamStore`, Core reads, appends and closes a Stream itself, with no Activity. A read is `iter` of the Workflow -`Stream`. It gives each Record as it arrives and ends at a stop. By default a read has no -bound, so it reads until the Stream is closed and the read gave its last Record. +`Stream`. It gives each Record as it arrives and ends at a stop. By default a read ends with +`Idle` after 1 minute with no new Record, and has no other bound. Here the Workflow reads the events of a model's answer while the model Activity still runs, and starts each tool as its call arrives. @@ -71,11 +71,14 @@ async for record in events.iter(): results = await asyncio.gather(*tools) ``` +When the Stream can be quiet for more than a minute, as a model that thinks for long can be, +check `done` after the loop, or set a longer `idle_timeout`. + `iter` takes `from_offset` and four bounds, `timeout`, `idle_timeout`, `max_records` and -`max_bytes`. Each bound is `None` by default, which is no bound. An `idle_timeout` ends a -read with `Idle` when no Record comes for that long, and zero gives what is in the Stream -and doesn't wait. After a bound ended a read, a new read from `last_offset` goes on where it -stopped. This loop waits with a timer while the Stream is quiet. +`max_bytes`. `idle_timeout` is 1 minute by default, and the others have no bound. An +`idle_timeout` ends a read with `Idle` when no Record comes for that long, and zero gives +what is in the Stream and doesn't wait. After a bound ended a read, a new read from +`last_offset` goes on where it stopped. This loop waits with a timer while the Stream is quiet. ```python last = None @@ -102,22 +105,27 @@ of Records. It has three other costs. - A read keeps its Workflow Task open. A Signal, an Update, a cancel or an Activity result reaches the Workflow at the next Workflow Task, at the heartbeat of the task at the latest. That is about 8 s with the default task timeout of 10 s. A read that waits on a quiet - Stream holds its task open the whole time, with one Workflow Task about every 8 s. An - `idle_timeout` ends such a read with `Idle`. + Stream holds its task open the whole time, with one Workflow Task about every 8 s, until + its `idle_timeout` ends it with `Idle`. - By default, `StreamReplaySource.STORE`, the marker holds a range of offsets, a count and a digest, and no Record. A replay reads the Store again, so the Records must stay in the Store for as long as the Workflow can replay. The option `workflow_read_replay_from=StreamReplaySource.HISTORY` of `StreamConfig` puts the Records into the marker, so a replay needs no Store. Then a read with no `max_bytes` gets a cap of - 256 KiB. + 256 KiB, and one Workflow Task's marker holds at most 1 MiB of Records. Past that, the task + completes and the read goes on in the next one. - A replay on a Worker with no cache reads the Store again on every Workflow Task. While Workflow code is busy, a read holds at most 4 MiB of its Records in the Worker, then waits until the code takes them. `workflow_read_ahead_bytes` of `StreamConfig` changes that -bound, for large Records or a Worker with little memory. +bound, for large Records or a Worker with little memory. A read gives Workflow code a batch at +most once in 50 ms, so a busy Stream doesn't activate the Workflow for each Record. +`workflow_read_batch_window` of `StreamConfig` changes that window. An append and a close of Workflow code always write as the Workflow, with the Workflow Task as -their epoch. An append runs inside its Workflow Task and records a marker with no body. +their epoch. An append runs inside its Workflow Task and records a marker with no body. An +append is at-least-once: when a Worker crashes or its run is evicted before the task with the +append completes, the rerun of that task writes a second copy with the epoch of the new task. A close spans two Workflow Tasks. The task that calls `close` records only the intent. Core closes the Stream after the server took that task, and the next task gives the result. So a @@ -158,5 +166,6 @@ counts nothing. | `temporal_stream_append_latency` | From the start of an append in Core to its result | | `temporal_stream_close_total`, `_failed` | Closes that Core sent, and those that failed | | `temporal_stream_close_latency` | From the start of a close in Core to its result | +| `temporal_stream_close_abandoned` | Closes that a graceful shutdown gave up after 10 s | | `temporal_stream_replay_read_total`, `_failed` | Marker entries that a replay read again | | `temporal_stream_replay_read_latency` | The time of one such read and its check | diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index d6290dbb2..54c2e495a 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -203,8 +203,9 @@ class StreamReadStop(enum.IntEnum): Idle = 4 """No Record came for ``idle_timeout`` while the read waited for the Store. - Only a read with an ``idle_timeout`` ends with this stop. A read with none waits on a - quiet Stream until another stop holds. + A read with no ``idle_timeout`` ends with this stop after 1 minute with no Record. So + code that reads a Stream that can be quiet for longer checks ``done`` after the loop, + or sets a longer ``idle_timeout``. """ Timeout = 5 @@ -376,8 +377,8 @@ def iter_stream( It is the ``iter`` of a Workflow Stream. It does no I/O and sends nothing, so it takes no ``await``. The read starts at the first step of the ``async for`` loop. Core then reads the Store inside the Workflow Task and gives the Records to the loop - as they arrive. By default a read has no bound, so it reads until the Stream is - closed and the read gave its last Record. The loop ends at the first stop. Then + as they arrive. By default a read ends with ``Idle`` after 1 minute with no new + Record, and has no other bound. The loop ends at the first stop. Then :py:attr:`StreamReading.stop` tells which stop it was, and :py:attr:`StreamReading.done` is true when the Stream is closed and the read gave its last Record. That stop is ``Done`` also when the last Record filled a cap. A new @@ -404,8 +405,11 @@ def iter_stream( await status.append(f"Running {event.tool_call.name}") results = await asyncio.gather(*tools) - A cap, a ``timeout`` or an ``idle_timeout`` ends a read before the Stream is done. - A loop of reads from ``last_offset`` then goes on where the last read stopped. + A loop ends before the Stream is done when the Stream is quiet for the + ``idle_timeout``, or when a cap or a ``timeout`` holds. When the Stream can be quiet + for more than a minute, as a model that thinks for long can be, check ``done`` after + the loop or set a longer ``idle_timeout``. A loop of reads from ``last_offset`` goes + on where the last read stopped. .. code-block:: python @@ -428,7 +432,7 @@ def iter_stream( Task. That is when the read ends, or at the heartbeat of the task at the latest, which is about 8 s with the default task timeout of 10 s. A read that waits on a quiet Stream holds its task open the whole time, with one Workflow Task about - every 8 s. An ``idle_timeout`` ends such a read with ``Idle``. + every 8 s, until its ``idle_timeout`` ends it with ``Idle``. * With the default :py:attr:`temporalio.client.StreamReplaySource.STORE`, History holds no Record. A replay reads the Records from the Store again, so they must stay in the Store for as long as the Workflow can replay. When they are gone, the @@ -458,8 +462,8 @@ def iter_stream( Stream. idle_timeout: The read ends with ``Idle`` when no Record comes for this long while it waits for the Store. Time in which its Records wait for the loop - doesn't count. ``None`` is no bound, so the read waits for the next Record. - Zero gives what is in the Stream and doesn't wait. + doesn't count. ``None`` is 1 minute. Zero gives what is in the Stream and + doesn't wait. max_records: The most Records of the read. ``None`` is no cap. max_bytes: The most bytes of the Records of the read, each as its encoded ``StreamRecord``. The first Record always comes. ``None`` is no cap, unless From d0bfe02bfece5bc812c87d304fd9a13b647a5bb5 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 13:43:19 -0700 Subject: [PATCH 44/59] Tested the batch window and a Workflow failure for lost Records. A window of 1 s joins the four later Records of a read in one batch, and a zero window gives more batches. Lost Records fail the task by default, or the Workflow with the setting. --- tests/contrib/redis/test_streams.py | 2 +- tests/nexus/test_stream_direct.py | 173 +++++++++++++++++++++++++++- 2 files changed, 171 insertions(+), 4 deletions(-) diff --git a/tests/contrib/redis/test_streams.py b/tests/contrib/redis/test_streams.py index ae23a9981..6dd5c9325 100644 --- a/tests/contrib/redis/test_streams.py +++ b/tests/contrib/redis/test_streams.py @@ -209,7 +209,7 @@ async def run(self, stream_id: str, seen_id: str) -> list[str]: stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) seen = workflow.get_stream(StreamRef(id=seen_id), body_type=str) got: list[str] = [] - # A read has no idle bound, so one read waits across the Activity's two steps. + # The idle default of 1 minute is far longer than the pause between the two steps. reading = stream.iter() async for record in reading: got.append(record.body) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index c46bf5c76..7427a96fd 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -31,7 +31,7 @@ WithSerializationContext, WorkflowSerializationContext, ) -from temporalio.exceptions import ApplicationError, StreamError +from temporalio.exceptions import ApplicationError, StreamError, StreamReplayError from temporalio.nexus.system.stream_service import ( Producer, ProducerKindWorkflow, @@ -42,7 +42,7 @@ from temporalio.runtime import MetricBuffer, Runtime, TelemetryConfig from temporalio.testing import MemoryStreamStore, WorkflowEnvironment from temporalio.worker import Replayer, Worker -from tests.helpers import new_worker +from tests.helpers import assert_eventually, assert_task_fail_eventually, new_worker from tests.helpers.streams import DictStreamStore from tests.test_converter import SimpleCodec @@ -169,7 +169,7 @@ async def stream_model_output(call: ModelCall) -> None: async def model_chunks(chunks_id: str) -> AsyncGenerator[str, None]: """Yield each chunk of the model's Stream, as the model stub of a library does.""" stream = workflow.get_stream(StreamRef(id=chunks_id), body_type=str) - # A read has no idle bound, so one read waits for the model until it closes the Stream. + # The idle default of 1 minute is far longer than a pause of the model, so one read waits. async for record in stream.iter(): yield record.body @@ -1949,3 +1949,170 @@ async def test_read_with_a_read_ahead_of_one_byte_gives_every_record(client: Cli _, result = await _run(stream_client, ReadOnceWorkflow, ReadArgs(ref.id)) assert result.bodies == bodies assert (result.stop, result.done) == ("Done", True) + + +# The batch window. + + +def _read_page_counts(history: WorkflowHistory) -> list[int]: + """Give the Records of each batch of the read entries of the markers, in order.""" + counts: list[int] = [] + for event in history.events: + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + for entry in StreamMarkerData.FromString(payload.data).reads: + counts.extend(entry.page_counts) + return counts + + +@activity.defn +async def append_five_apart(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + for index in range(5): + if index: + await asyncio.sleep(0.05) + await stream.append(f"record-{index}") + await stream.close() + + +@workflow.defn +class WindowedReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + # A Local Activity holds the Workflow Task open, so each batch of the read comes + # in an activation of this task and not in a later task. + writer = workflow.start_local_activity( + append_five_apart, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [record.body async for record in stream.iter()] + await writer + return got + + +async def _windowed_read(client: Client, window: timedelta) -> list[int]: + config = client.config() + config["stream_config"] = StreamConfig( + stores=[MemoryStreamStore()], workflow_read_batch_window=window + ) + stream_client = Client(**config) + ref = await _filled_stream(stream_client, [], close=False) + handle, got = await _run( + stream_client, WindowedReadWorkflow, ref.id, activities=[append_five_apart] + ) + assert got == [f"record-{index}" for index in range(5)] + return _read_page_counts(await handle.fetch_history()) + + +async def test_batch_window_joins_the_records_of_a_window(client: Client): + # The first batch comes immediately, and the window holds the other four for 1 s. + assert await _windowed_read(client, timedelta(seconds=1)) == [1, 4] + + +async def test_zero_batch_window_gives_more_batches(client: Client): + assert len(await _windowed_read(client, timedelta())) > 2 + + +def test_negative_batch_window_raises(): + with pytest.raises(ValueError, match="workflow_read_batch_window"): + StreamConfig( + stores=[MemoryStreamStore()], + workflow_read_batch_window=timedelta(milliseconds=-1), + ) + + +# A replay that lost the Records of a read. + + +class _ReadThenWait: + def __init__(self) -> None: + self.go = False + + async def read_then_wait(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [record.body async for record in stream.iter(idle_timeout=timedelta())] + await workflow.wait_condition(lambda: self.go) + return got + + @workflow.signal + def proceed(self) -> None: + self.go = True + + +@workflow.defn +class ReadThenWaitWorkflow(_ReadThenWait): + @workflow.run + async def run(self, stream_id: str) -> list[str]: + return await self.read_then_wait(stream_id) + + +@workflow.defn(failure_exception_types=[StreamReplayError]) +class ReadThenWaitFailsWorkflow(_ReadThenWait): + @workflow.run + async def run(self, stream_id: str) -> list[str]: + return await self.read_then_wait(stream_id) + + +async def _replay_with_records_gone( + client: Client, workflow_type: type[Any], **worker_options: Any +) -> WorkflowHandle[Any, Any]: + """Read on one Worker, then give the run to a Worker whose Store lost the Records.""" + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b"]) + task_queue = f"tq-{uuid.uuid4()}" + # No cache, so the next Worker replays the History from the start. + async with new_worker( + stream_client, workflow_type, task_queue=task_queue, max_cached_workflows=0 + ): + handle: WorkflowHandle[Any, Any] = await stream_client.start_workflow( + workflow_type.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=task_queue, + ) + + async def read_is_in_history() -> None: + assert "core_stream" in _markers(await handle.fetch_history()) + + await assert_eventually(read_is_in_history) + # A new memory Store of the same name has none of the Records. + lost_client = _stream_client(client) + worker = new_worker( + lost_client, workflow_type, task_queue=task_queue, **worker_options + ) + async with worker: + await handle.signal("proceed") + if workflow_type is ReadThenWaitWorkflow and not worker_options: + await assert_task_fail_eventually( + handle, message_contains=f"StreamReplayError: Stream `{ref.id}`" + ) + await handle.terminate() + return handle + with pytest.raises(WorkflowFailureError) as err: + await asyncio.wait_for(handle.result(), timeout=30) + cause = err.value.cause + assert isinstance(cause, ApplicationError) + assert cause.type == "StreamReplayError" + assert f"StreamReplayError: Stream `{ref.id}`" in cause.message + return handle + + +async def test_lost_records_fail_the_task_by_default(client: Client): + await _replay_with_records_gone(client, ReadThenWaitWorkflow) + + +async def test_lost_records_fail_the_workflow_with_the_worker_option(client: Client): + await _replay_with_records_gone( + client, + ReadThenWaitWorkflow, + workflow_failure_exception_types=[StreamReplayError], + ) + + +async def test_lost_records_fail_the_workflow_with_the_defn_option(client: Client): + await _replay_with_records_gone(client, ReadThenWaitFailsWorkflow) From 7325df1e38afcb5972091e6a192f5808b2a4d037 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 23:20:35 -0700 Subject: [PATCH 45/59] Moved Core to the Stream WIT with one read for both targets. Core vendors the WIT that has no separate Workflow read, and its Store option is called `producer_idle_ttl`, as on the base branch. --- temporalio/bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 2c1c85fe2..2e8f906cf 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 2c1c85fe2ff0580fbedd5d6a7f14d738372fcff4 +Subproject commit 2e8f906cff399bcf3def95acbcec5e4366adf568 From e29e33cc9d2d0e154f53c40d1d7a486df2208691 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 23:20:45 -0700 Subject: [PATCH 46/59] Said that a close on a Python Store runs in a Local Activity. The base branch runs a Workflow's close on a Store written in Python in a Local Activity, so the docs of this branch follow it. --- temporalio/client/_stream_store.py | 6 +++--- temporalio/worker/_replayer.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index aabd41d9a..132a35f09 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -108,9 +108,9 @@ class StreamStore(ABC): The SDK's Stores run on the bridge, in Rust, and this class isn't meant to be subclassed in Python. A Store that is written in Python works for a client and an Activity, for example as a double in a test. Workflow code loses the Direct calls on - it. An append of Workflow code then runs in a Local Activity, with one marker in - History for each call, a close runs in a regular Activity after its Workflow Task, - and ``iter`` fails the Workflow, so Workflow code can't read its Streams. + it. An append and a close of Workflow code then run in a Local Activity, with one + marker in History for each call, and ``iter`` fails the Workflow, so Workflow code + can't read its Streams. Such a Store implements :py:attr:`name` and the five operations. Each operation takes and gives a message of diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index b60948103..cf886c8c2 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -73,15 +73,15 @@ def __init__( ``stream_config`` takes the Stream config of the client that the worker had, so a replay makes the same commands. A history that records a - create, a describe, or an append or a close through a Store's Local Activity needs - it. With no Stream config, Workflow code gets an error in place of that call, - and the replay fails with a nondeterminism error. A replay calls no method of - its Stores from Python. An append or a close that Core made needs no Stream + create, a describe, or an append or a close through a Store's Local Activity + needs it. With no Stream config, Workflow code gets an error in place of that + call, and the replay fails with a nondeterminism error. A replay calls no method + of its Stores from Python. An append or a close that Core made needs no Stream config, and a replay closes nothing. A read of Workflow code with ``iter`` that ran under the option ``HISTORY`` of ``workflow_read_replay_from`` needs none either. That holds also when the ref names no Store, since Core takes the Store - from the history. A read that ran under the default, ``StreamReplaySource.STORE``, has no - Records in the history. Core reads them from the Store again, so the replay + from the history. A read that ran under the default, + ``StreamReplaySource.STORE``, has no Records in the history. Core reads them from the Store again, so the replay needs this config and Stores that still have the Records. Without them the replay fails with a :py:class:`RuntimeError` whose message has ``StreamReplayError``, the Stream and the cause. From f66743f35ef8b9b5b31eef9390f559c2ac1848cf Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 23:22:05 -0700 Subject: [PATCH 47/59] Fitted the close tests to the Local Activity close of the base branch. A close on a Python Store runs in a Local Activity. The Direct close test of a failed task needs a Workflow that fails in the task of the intent. --- tests/nexus/test_stream_direct.py | 6 +++-- tests/nexus/test_stream_store_workflow.py | 33 ++++++++++++++++++----- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 7427a96fd..067fa8258 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -1760,13 +1760,15 @@ async def test_cancel_while_the_close_waits(client: Client): assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed -async def test_python_store_closes_through_its_activity(client: Client): +async def test_python_store_closes_through_its_local_activity(client: Client): stream_client = _stream_client(client, DictStreamStore()) ref = await _filled_stream(stream_client, [], close=False) handle, _ = await _run(stream_client, AppendMaybeCloseWorkflow, ref.id, True) assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed history = await handle.fetch_history() - assert _scheduled_activities(history) == ["__temporal_stream_store_call:dict"] + assert _scheduled_activities(history) == [] + # One marker for the append and one for the close. + assert _markers(history) == ["core_local_activity"] * 2 assert _close_entries(history) == [] diff --git a/tests/nexus/test_stream_store_workflow.py b/tests/nexus/test_stream_store_workflow.py index 5e952060d..053ecaddc 100644 --- a/tests/nexus/test_stream_store_workflow.py +++ b/tests/nexus/test_stream_store_workflow.py @@ -395,6 +395,26 @@ async def run(self, stream_id: str) -> str: return "closed" +_fail_close_intent_task = True + + +@workflow.defn(sandboxed=False) +class CloseIntentThenFailTaskOnceWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + stream = get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + close = asyncio.create_task(stream.close()) + # A few turns of the loop let the close send its intent in this task. + for _ in range(10): + await asyncio.sleep(0) + global _fail_close_intent_task + if _fail_close_intent_task: + _fail_close_intent_task = False + raise RuntimeError("Failed the Workflow Task once") + await close + + class RecordingInterceptor(Interceptor): def __init__(self) -> None: self.activity_types: list[str] = [] @@ -1195,7 +1215,7 @@ async def test_failed_workflow_task_after_close_leaves_the_stream_closed( global _fail_close_task _fail_close_task = True # A Store written in Python, since Core closes a Stream on a Store with a runtime. - stream_client = _with_store(client, DictStreamStore()) + stream_client = _with_store(client, RecordingStreamStore(MemoryStreamStore())) stream = await stream_client.create_stream( id=f"stream-{uuid.uuid4()}", body_type=str ) @@ -1226,22 +1246,21 @@ async def test_failed_workflow_task_after_close_leaves_the_stream_closed( async def test_failed_workflow_task_after_direct_close_leaves_the_stream_open( client: Client, ): - global _fail_close_task - _fail_close_task = True + global _fail_close_intent_task + _fail_close_intent_task = True stream_client = _with_store(client, MemoryStreamStore()) stream = await stream_client.create_stream( id=f"stream-{uuid.uuid4()}", body_type=str ) - async with new_worker(stream_client, CloseThenFailTaskOnceWorkflow) as worker: - result = await stream_client.execute_workflow( - CloseThenFailTaskOnceWorkflow.run, + async with new_worker(stream_client, CloseIntentThenFailTaskOnceWorkflow) as worker: + await stream_client.execute_workflow( + CloseIntentThenFailTaskOnceWorkflow.run, stream.ref.id, id=f"workflow-{uuid.uuid4()}", task_queue=worker.task_queue, ) # Core makes the close with no Activity. The failed task recorded no intent, so it # closed nothing, and the retried task appends again before it closes the Stream. - assert result == "closed" records = await _read_all(stream_client, stream.ref.id) assert [record.body for record in records] == ["a", "a"] assert await _describe_status(stream_client, stream.ref.id) == ( From 3129f1f36aa19325ee7c598b81db147fabb7907b Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 23:22:44 -0700 Subject: [PATCH 48/59] Pinned the nexgen fork commit with the Workflow poll iterators. The Workflow `Stream` reads with the same `read`, `iter_read` and `iter_read_items` as a client, so the generated Workflow target has no `iter`. --- .github/workflows/ci.yml | 3 +- scripts/gen_nexus_system_api.py | 5 +- .../system/stream_service/workflow/_poll.py | 77 +++++++++++ .../workflow/_resources/stream.py | 127 ++++++++++++++++-- 4 files changed, 195 insertions(+), 17 deletions(-) create mode 100644 temporalio/nexus/system/stream_service/workflow/_poll.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8804392a3..72aec058b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,7 +229,8 @@ jobs: - run: uv sync --all-extras - run: cargo install --locked nexgen --version 0.2.8 --features advanced --force # The Stream package needs `@nexus.helper` with `sync=true`, which only a nexgen fork has. - - run: cargo install --locked --git https://github.com/moetemp/nexgen --rev e4b1f297e7a3fa8bc20fbf176fa2727c9e83a20f --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen + # This commit of the fork also has the poll iterators of the Workflow target. + - run: cargo install --locked --git https://github.com/moetemp/nexgen --rev 2785462ecdaba5da7a27051fbaaccbaee26ae9e0 --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen - run: echo "NEX_GEN_STREAM_BIN=$RUNNER_TEMP/nexgen-stream/bin/nexgen" >> "$GITHUB_ENV" - run: poe build-develop - run: poe gen-protos diff --git a/scripts/gen_nexus_system_api.py b/scripts/gen_nexus_system_api.py index 4c725250a..b62c423ef 100644 --- a/scripts/gen_nexus_system_api.py +++ b/scripts/gen_nexus_system_api.py @@ -30,7 +30,8 @@ # The Stream package needs a newer nexgen, which rejects the other WITs. One pin is enough # again once the api WITs and a nexgen release agree. The Stream WIT uses `@nexus.helper` with # `sync=true` and `@nexus.support-type`, which only the fork https://github.com/moetemp/nexgen -# has, at commit e4b1f297. So NEX_GEN_STREAM_BIN points at a build of that commit. +# has. Its commit 2785462e also has the poll iterators of the Workflow target. So +# NEX_GEN_STREAM_BIN points at a build of that commit. NEX_GEN_STREAM_VERSION = "0.2.12" proto_files = [ gen_protos.api_proto_dir @@ -88,7 +89,7 @@ def nex_gen_stream_command() -> list[str]: bin_path = os.environ.get("NEX_GEN_STREAM_BIN") if not bin_path: raise RuntimeError( - "NEX_GEN_STREAM_BIN must point at a nexgen build of commit e4b1f297 " + "NEX_GEN_STREAM_BIN must point at a nexgen build of commit 2785462e " "of https://github.com/moetemp/nexgen" ) version = subprocess.check_output([bin_path, "--version"], text=True).strip() diff --git a/temporalio/nexus/system/stream_service/workflow/_poll.py b/temporalio/nexus/system/stream_service/workflow/_poll.py new file mode 100644 index 000000000..a653d84f9 --- /dev/null +++ b/temporalio/nexus/system/stream_service/workflow/_poll.py @@ -0,0 +1,77 @@ +# Generated by nexgen v0.2.12. DO NOT EDIT! + +from __future__ import annotations + +import collections +import collections.abc +import typing + +_PollRequestT = typing.TypeVar("_PollRequestT") +_PollResponseT = typing.TypeVar("_PollResponseT") +_PollItemT = typing.TypeVar("_PollItemT") + + +@typing.final +class PollIterator(typing.Generic[_PollRequestT, _PollItemT]): + """Yields each item of the pages of a poll. + + ``next_request`` is the request to resume with. After an item, the resume helper gives + it. Without a resume helper, it reads the item's page again. After a page's last item, + it is the page helper's request, which is ``None`` after the last page. + """ + + def __init__( + self, + request: _PollRequestT, + pages: collections.abc.AsyncIterator[_PollResponseT], + items: collections.abc.Callable[ + [_PollResponseT], collections.abc.Sequence[_PollItemT] | None + ], + next_page: collections.abc.Callable[ + [_PollRequestT, _PollResponseT], _PollRequestT | None + ], + resume: collections.abc.Callable[[_PollRequestT, _PollItemT], _PollRequestT] + | None, + ) -> None: + async def read( + page_request: _PollRequestT, + ) -> tuple[_PollRequestT | None, collections.deque[_PollItemT]]: + response = await anext(pages) + # The page iterator keeps its requests to itself, so the page helper runs here + # again. It runs before the caller gets an item, so a changed item can't change + # the next request. + return next_page(page_request, response), collections.deque( + items(response) or () + ) + + self.next_request: _PollRequestT | None = request + self._read: collections.abc.Callable[ + [_PollRequestT], + collections.abc.Awaitable[ + tuple[_PollRequestT | None, collections.deque[_PollItemT]] + ], + ] = read + self._resume: ( + collections.abc.Callable[[_PollRequestT, _PollItemT], _PollRequestT] | None + ) = resume + self._page: collections.deque[_PollItemT] = collections.deque() + self._page_request: _PollRequestT = request + self._page_next: _PollRequestT | None = None + + def __aiter__(self) -> PollIterator[_PollRequestT, _PollItemT]: + return self + + async def __anext__(self) -> _PollItemT: + while not self._page: + if self.next_request is None: + raise StopAsyncIteration + self._page_request = self.next_request + self._page_next, self._page = await self._read(self._page_request) + if not self._page: + self.next_request = self._page_next + item = self._page.popleft() + if not self._page: + self.next_request = self._page_next + elif self._resume is not None: + self.next_request = self._resume(self._page_request, item) + return item diff --git a/temporalio/nexus/system/stream_service/workflow/_resources/stream.py b/temporalio/nexus/system/stream_service/workflow/_resources/stream.py index 36018dbd2..0aa32cc38 100644 --- a/temporalio/nexus/system/stream_service/workflow/_resources/stream.py +++ b/temporalio/nexus/system/stream_service/workflow/_resources/stream.py @@ -2,6 +2,7 @@ from __future__ import annotations +import collections.abc import datetime import types import typing @@ -21,8 +22,12 @@ DescribeStreamRequest, DescribeStreamResponse, Producer, + ReadRecordsRequest, + ReadRecordsResponse, + StreamRecord, StreamRef, ) +from .._poll import PollIterator BodyT = typing.TypeVar("BodyT") @@ -87,33 +92,112 @@ async def close( handle = await _close_stream(request) await handle - async def describe( + async def read( self, - ) -> DescribeStreamResponse: - """Describe the Stream's status and newest offset.""" - request = DescribeStreamRequest( + from_offset: bytes | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + ) -> ReadRecordsResponse[BodyT]: + """Read one page of Records. The poll iterator reads them all.""" + request = ReadRecordsRequest( ref=self.ref, + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, ) - handle = await _describe_stream(request) + handle = await _read_records(request, self.body_type) return await handle - def iter( + async def iter_read( + self, + from_offset: bytes | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + ) -> collections.abc.AsyncGenerator[ReadRecordsResponse[BodyT], None]: + """Yield each response of ``read`` until the poll helper stops. + + Read one page of Records. The poll iterator reads them all. + + Args: + from_offset: Read after this offset. Empty is the beginning. A too high + offset fails. + max_records: About this many Records per page, 1 to 1000. + wait: How long to wait for a new Record before an empty answer. At most 60 + s. + max_bytes: The maximum Record bytes in a page. A larger Record is still + returned alone. + """ + request = ReadRecordsRequest( + ref=self.ref, + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, + ) + while True: + handle = await _read_records(request, self.body_type) + response = await handle + next_request = _support.next_read_request(request, response) + yield response + if next_request is None: + return + request = next_request + + def iter_read_items( self, - timeout: datetime.timedelta | None = None, from_offset: bytes | None = None, - idle_timeout: datetime.timedelta | None = None, max_records: int | None = None, + wait: datetime.timedelta | None = None, max_bytes: int | None = None, - ) -> _support.StreamReading[BodyT]: - """Read each Record as it arrives, up to a stop.""" - return _support.iter_stream( - self, - timeout=timeout, + ) -> PollIterator[ReadRecordsRequest[BodyT], StreamRecord[BodyT]]: + """Yield each item of ``iter_read``, page by page. + + ``next_request`` resumes after the last item that the iterator yielded. It is + ``None`` after the last page. + + Read one page of Records. The poll iterator reads them all. + + Args: + from_offset: Read after this offset. Empty is the beginning. A too high + offset fails. + max_records: About this many Records per page, 1 to 1000. + wait: How long to wait for a new Record before an empty answer. At most 60 + s. + max_bytes: The maximum Record bytes in a page. A larger Record is still + returned alone. + """ + request = ReadRecordsRequest( + ref=self.ref, from_offset=from_offset, - idle_timeout=idle_timeout, max_records=max_records, + wait=wait, max_bytes=max_bytes, ) + return PollIterator( + request, + self.iter_read( + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, + ), + lambda response: response.records, + _support.next_read_request, + _support.resume_read_request, + ) + + async def describe( + self, + ) -> DescribeStreamResponse: + """Describe the Stream's status and newest offset.""" + request = DescribeStreamRequest( + ref=self.ref, + ) + handle = await _describe_stream(request) + return await handle @typing.overload @@ -183,6 +267,21 @@ async def _close_stream( ) +async def _read_records( + request: ReadRecordsRequest[BodyT], + body_type: type[BodyT], +) -> temporalio.workflow.NexusOperationHandle[ReadRecordsResponse[BodyT],]: + nexus_client = temporalio.workflow.create_nexus_client( + service="temporal.api.streamservice.v1.StreamService", + endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, + ) + return await nexus_client.start_operation( + operation="ReadRecords", + input=request, + output_type=ReadRecordsResponse[body_type], + ) + + async def _describe_stream( request: DescribeStreamRequest[BodyT], ) -> temporalio.workflow.NexusOperationHandle[DescribeStreamResponse,]: From ec7303b131ef6b9a5aed759a90c1dcc52565007a Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 23:37:31 -0700 Subject: [PATCH 49/59] Served each Workflow read call from Core's long-lived read. Workflow code reads with the same `read`, `iter_read` and `iter_read_items` as a client, so the separate `iter` and `StreamReading` go. A Core read that no call waits on closes before the activation completes, so it never holds a Workflow Task open for nobody. --- temporalio/client/_stream_store.py | 11 +- temporalio/common.py | 13 - temporalio/nexus/system/_support.py | 327 --------- temporalio/worker/_stream_store.py | 248 +++++-- temporalio/worker/_workflow_instance.py | 157 ++--- temporalio/workflow/__init__.py | 5 - temporalio/workflow/_context.py | 6 - tests/contrib/redis/test_streams.py | 29 +- tests/nexus/test_stream_direct.py | 791 +++++++++++++--------- tests/nexus/test_stream_store_workflow.py | 58 +- 10 files changed, 807 insertions(+), 838 deletions(-) diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index 132a35f09..e6a97b396 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -42,8 +42,8 @@ class StreamReplaySource(enum.Enum): """The marker in History holds the Records of the read with their bodies. A replay takes the Records from History, so it reads no Store and needs no Stream - config. Each Record adds to History, so a read with no ``max_bytes`` gets the cap - :py:data:`temporalio.common.DEFAULT_STREAM_MAX_BYTES`. + config. Each Record adds to History. The marker of one Workflow Task holds at most + 1 MiB of Records, and a read goes on in the next task after that. """ STORE = "store" @@ -325,9 +325,6 @@ class _StreamOperation: response_type: type[google.protobuf.message.Message] - workflow_refusal: str | None = None - """Why Workflow code can't start it in this release, or ``None`` when it can.""" - async def call( self, store: StreamStore, request: google.protobuf.message.Message ) -> google.protobuf.message.Message: @@ -365,10 +362,6 @@ def check_response( "read_records", temporalio.api.streamservice.v1.ReadRecordsRequest, temporalio.api.streamservice.v1.ReadRecordsResponse, - workflow_refusal=( - "Workflow code reads a Stream with `iter` of the Workflow Stream, not " - "with the ReadRecords operation." - ), ), "CloseStream": _StreamOperation( "close_stream", diff --git a/temporalio/common.py b/temporalio/common.py index af1af74e4..8bcc66a4a 100644 --- a/temporalio/common.py +++ b/temporalio/common.py @@ -1455,19 +1455,6 @@ class HeaderCodecBehavior(IntEnum): """Only automatically encode and decode headers in workflow activation encoding and decoding.""" -DEFAULT_STREAM_MAX_BYTES = 256 * 1024 -"""The most bytes of the Records of one read of a Stream in Workflow code, when History -holds the Records. - -It applies when the read's ``max_bytes`` is ``None``. By default a read of Workflow code has -no such cap, since no body goes into History. It has this cap when -``StreamConfig.workflow_read_replay_from`` is the option ``HISTORY``. - -.. warning:: - This API is experimental and subject to change. -""" - - @dataclass class _CompositeEvent: # pyright: ignore[reportUnusedClass] # This should always be present, but is sometimes lazily set internally diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index 54c2e495a..55f53b59a 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -1,8 +1,5 @@ -import asyncio -import collections import collections.abc import dataclasses -import enum import functools import typing import urllib.parse @@ -11,7 +8,6 @@ import google.protobuf.duration_pb2 import google.protobuf.timestamp_pb2 -import typing_extensions import temporalio.activity import temporalio.api.common.v1.message_pb2 as common_pb2 @@ -24,7 +20,6 @@ import temporalio.nexus.system if typing.TYPE_CHECKING: - import temporalio.api.stream.v1 from temporalio.nexus.system.stream_service.models import ( Producer, ReadRecordsRequest, @@ -34,11 +29,6 @@ _BodyT = typing.TypeVar("_BodyT") -if typing.TYPE_CHECKING: - _RecordDecoder: typing.TypeAlias = collections.abc.Callable[ - [temporalio.api.stream.v1.StreamRecord], StreamRecord[_BodyT] - ] - class SignalWithStartWorkflowModelRequest(typing.Protocol): namespace: str @@ -180,323 +170,6 @@ def resume_read_request( return dataclasses.replace(request, from_offset=item.offset) -class StreamReadStop(enum.IntEnum): - """Why a read of Workflow code ended. - - .. warning:: - This API is experimental and subject to change. - """ - - Done = 1 - """The Stream is closed, and the read gave its last Record. - - A read that gave the last Record of a closed Stream ends with this stop, also when - that Record filled ``max_records`` or ``max_bytes``. - """ - - MaxRecords = 2 - """The read gave ``max_records`` Records, and the Stream has more or is open.""" - - MaxBytes = 3 - """The next Record would pass ``max_bytes``.""" - - Idle = 4 - """No Record came for ``idle_timeout`` while the read waited for the Store. - - A read with no ``idle_timeout`` ends with this stop after 1 minute with no Record. So - code that reads a Stream that can be quiet for longer checks ``done`` after the loop, - or sets a longer ``idle_timeout``. - """ - - Timeout = 5 - """``timeout`` passed since the read started.""" - - -@dataclasses.dataclass(frozen=True) -class _StreamReadOptions: - """The checked arguments of one read, as `iter_stream` takes them.""" - - from_offset: bytes | None - timeout: timedelta | None - idle_timeout: timedelta | None - max_records: int | None - max_bytes: int | None - - -class StreamReading(typing.Generic[_BodyT]): - """One read of a Stream in Workflow code. It gives each Record as it arrives. - - Get it from ``iter`` of a Workflow Stream, and read it with ``async for``. The read - starts at the first step of the loop and ends at a stop. Then :py:attr:`stop` and - :py:attr:`done` tell why, and :py:attr:`last_offset` is the ``from_offset`` of the - next read. See ``iter`` for the stops and the costs of a read. - - .. warning:: - This API is experimental and subject to change. - """ - - last_offset: bytes | None - """The offset of the last Record that the loop got, or ``from_offset`` when it got - none. A new read from it loses and repeats no Record.""" - - stop: StreamReadStop | None - """Why the read ended. ``None`` while it runs, and after a failure or a close.""" - - done: bool - """Whether the Stream is closed and the read gave its last Record.""" - - def __init__(self, stream: typing.Any, options: _StreamReadOptions) -> None: - """Keep the call. Use ``iter`` of a Workflow Stream instead.""" - self._stream = stream - self._options = options - self.last_offset = options.from_offset - self.stop = None - self.done = False - self._seq: int | None = None - self._decode: ( - collections.abc.Callable[ - [temporalio.api.stream.v1.StreamRecord], StreamRecord[_BodyT] - ] - | None - ) = None - self._records: collections.deque[temporalio.api.stream.v1.StreamRecord] = ( - collections.deque() - ) - self._end: tuple[StreamReadStop | None, bool, BaseException | None] | None = ( - None - ) - # Each task that waits in `__anext__` has its own future, and a page, an end and a - # close wake every one of them. - self._waiters: list[asyncio.Future[None]] = [] - self._finished = False - self._error: BaseException | None = None - - def __aiter__(self) -> typing_extensions.Self: - """Give this read, since it is its own iterator.""" - return self - - async def __anext__(self) -> "StreamRecord[_BodyT]": - """Give the next Record, and start the read at the first call. - - Raises: - StopAsyncIteration: The read ended at a stop, or the code closed it. - temporalio.exceptions.StreamError: The read failed. Every later step raises - it again. - """ - import temporalio.workflow - - if self._finished: - self._raise_end() - if self._seq is None: - temporalio.workflow._Runtime.current().workflow_start_stream_read(self) - while not self._records and self._end is None and not self._finished: - waiter: asyncio.Future[None] = asyncio.get_running_loop().create_future() - self._waiters.append(waiter) - try: - await waiter - finally: - self._waiters.remove(waiter) - if self._finished: - self._raise_end() - if self._records: - record = self._records.popleft() - if self._decode is None: - raise RuntimeError("The Stream read has no decoder") - # A Record that fails to decode raises here, and `last_offset` stays before it, - # so a new read from `last_offset` gets it again. - decoded = self._decode(record) - self.last_offset = record.offset - return decoded - assert self._end is not None - stop, done, error = self._end - self._finished = True - self._error = error - self.stop = stop - self.done = done - self._raise_end() - - def _raise_end(self) -> typing.NoReturn: - # A second task that shares the read must not take a failure for a clean end, so - # every step after a failure raises it. - if self._error is not None: - raise self._error - raise StopAsyncIteration - - async def aclose(self) -> None: - """End a read that has started and has not ended, before a stop. - - Later steps of the loop end it. It does nothing on a read that has not started - or has ended. - """ - import temporalio.workflow - - if self._seq is None or self._finished: - return - temporalio.workflow._Runtime.current().workflow_close_stream_read( - self, send=self._end is None - ) - self._finished = True - self._records.clear() - self._wake() - - def _wake(self) -> None: - for waiter in self._waiters: - if not waiter.done(): - waiter.set_result(None) - - def _started( - self, - seq: int, - decode: "_RecordDecoder[_BodyT]", - ) -> None: - self._seq = seq - self._decode = decode - - def _take( - self, - records: "collections.abc.Iterable[temporalio.api.stream.v1.StreamRecord]", - end: tuple[StreamReadStop | None, bool, BaseException | None] | None, - ) -> None: - self._records.extend(records) - if end is not None: - self._end = end - self._wake() - - -def iter_stream( - stream: typing.Any, - *, - timeout: timedelta | None = None, - from_offset: bytes | None = None, - idle_timeout: timedelta | None = None, - max_records: int | None = None, - max_bytes: int | None = None, -) -> StreamReading[typing.Any]: - """Read each Record of a Stream as it arrives, up to a stop. - - It is the ``iter`` of a Workflow Stream. It does no I/O and sends nothing, so it - takes no ``await``. The read starts at the first step of the ``async for`` loop. - Core then reads the Store inside the Workflow Task and gives the Records to the loop - as they arrive. By default a read ends with ``Idle`` after 1 minute with no new - Record, and has no other bound. The loop ends at the first stop. Then - :py:attr:`StreamReading.stop` tells which stop it was, and - :py:attr:`StreamReading.done` is true when the Stream is closed and the read gave its - last Record. That stop is ``Done`` also when the last Record filled a cap. A new - read from :py:attr:`StreamReading.last_offset` loses and repeats no Record, also - after a close, a ``break``, or a Record whose body fails to decode. Such a Record - raises from its step of the loop, and ``last_offset`` stays before it. Two tasks can - step one read. Each Record goes to one of them, and both end when the read ends. A - read that failed raises its error at every step, in each task. - - .. code-block:: python - - tools = [] - async for record in events.iter(): - event = record.body - if event.kind == "tool_call": - # The tool starts now, before the model ends its answer. - tools.append( - workflow.start_activity( - run_tool, - event.tool_call, - start_to_close_timeout=timedelta(minutes=1), - ) - ) - await status.append(f"Running {event.tool_call.name}") - results = await asyncio.gather(*tools) - - A loop ends before the Stream is done when the Stream is quiet for the - ``idle_timeout``, or when a cap or a ``timeout`` holds. When the Stream can be quiet - for more than a minute, as a model that thinks for long can be, check ``done`` after - the loop or set a longer ``idle_timeout``. A loop of reads from ``last_offset`` goes - on where the last read stopped. - - .. code-block:: python - - last = None - while True: - reading = tokens.iter(from_offset=last, idle_timeout=timedelta(seconds=5)) - async for record in reading: - await shown.append(record.body.text) - last = reading.last_offset - if reading.done: - break - await workflow.sleep(timedelta(minutes=1)) - - A read costs one marker in History for each Workflow Task that it spans, whatever - the number of Records. A replay gives the loop the same Records at the same points. - A read has three other costs. - - * A read keeps its Workflow Task open, as a Local Activity does. A Signal, an - Update, a cancel or an Activity result reaches the Workflow at the next Workflow - Task. That is when the read ends, or at the heartbeat of the task at the latest, - which is about 8 s with the default task timeout of 10 s. A read that waits on a - quiet Stream holds its task open the whole time, with one Workflow Task about - every 8 s, until its ``idle_timeout`` ends it with ``Idle``. - * With the default :py:attr:`temporalio.client.StreamReplaySource.STORE`, History - holds no Record. A replay reads the Records from the Store again, so they must - stay in the Store for as long as the Workflow can replay. When they are gone, the - replay fails its Workflow Task with a message that starts with - ``StreamReplayError``. With :py:attr:`temporalio.client.StreamReplaySource.HISTORY`, - History holds the Records, and a replay needs no Store. - * A replay on a Worker with no cache reads the Store again on every Workflow Task. - - ``aclose`` of the read ends it before a stop. A read that the loop leaves with - ``break`` and doesn't close runs on in Core until one of its stops, and its Records - wait in the read. Call ``aclose`` when the code reads no more. - - The read needs a Store that runs on the bridge, as - :py:class:`temporalio.testing.MemoryStreamStore` and the Redis Store do. For a Store - written in Python, the first step of the loop fails the Workflow with a - non-retryable :py:class:`temporalio.exceptions.ApplicationError`. Code in a Query - handler or an Update validator can't read. - - Args: - stream: The Workflow Stream to read. - timeout: The longest time that the read takes, from the time it starts. Its - deadline holds across a replay, a failed Workflow Task and a Worker - restart. A read that gave no Record yet looks at the Store once before - its timeout ends it, so a short timeout still gives the Records that are - there. ``None`` is no bound. - from_offset: Read the Records after this offset. ``None`` is the start of the - Stream. - idle_timeout: The read ends with ``Idle`` when no Record comes for this long - while it waits for the Store. Time in which its Records wait for the loop - doesn't count. ``None`` is 1 minute. Zero gives what is in the Stream and - doesn't wait. - max_records: The most Records of the read. ``None`` is no cap. - max_bytes: The most bytes of the Records of the read, each as its encoded - ``StreamRecord``. The first Record always comes. ``None`` is no cap, unless - ``workflow_read_replay_from`` of the Stream config is ``HISTORY``. Then it is - :py:data:`temporalio.common.DEFAULT_STREAM_MAX_BYTES`. - - Returns: - The read, which the loop iterates. - - Raises: - ValueError: ``timeout``, ``max_records`` or ``max_bytes`` is zero or negative, - or ``idle_timeout`` is negative. - """ - if timeout is not None and timeout <= timedelta(): - raise ValueError("timeout must be positive") - if idle_timeout is not None and idle_timeout < timedelta(): - raise ValueError("idle_timeout must not be negative") - if max_records is not None and max_records < 1: - raise ValueError("max_records must be positive") - if max_bytes is not None and max_bytes < 1: - raise ValueError("max_bytes must be positive") - return StreamReading( - stream, - _StreamReadOptions( - from_offset=from_offset, - timeout=timeout, - idle_timeout=idle_timeout, - max_records=max_records, - max_bytes=max_bytes, - ), - ) - - def retry_policy_from_proto( proto: common_pb2.RetryPolicy, ) -> temporalio.common.RetryPolicy: diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index 798409d8b..0d5152bfe 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -3,20 +3,21 @@ from __future__ import annotations import asyncio +import collections import dataclasses -from collections.abc import Generator +from collections.abc import Generator, Iterable +from datetime import timedelta from typing import TYPE_CHECKING, Any import google.protobuf.message from nexusrpc import OutputT +import temporalio.api.enums.v1 import temporalio.api.failure.v1 import temporalio.api.stream.v1 import temporalio.api.streamservice.v1 import temporalio.bridge.proto.workflow_commands -import temporalio.common import temporalio.exceptions -import temporalio.nexus.system._support import temporalio.workflow from temporalio.client._stream_store import ( _STREAM_OPERATIONS, @@ -176,47 +177,194 @@ def _start_replayed_direct_call( # pyright: ignore[reportUnusedFunction] instance: _WorkflowInstanceImpl, input: StartNexusOperationInput[Any, OutputT], ) -> temporalio.workflow.NexusOperationHandle[OutputT] | None: - """Replay a Direct append or close on a Worker with no Stream config, or give ``None``. + """Replay a Direct call on a Worker with no Stream config, or give ``None``. - The marker of a Direct call holds its result, so its replay needs no Store. A ref that - names no Store sends an empty name, and Core takes the Store of History. + The marker of an append or a close holds its result, and the marker of a read can hold + its Records, so the replay needs no Store. A ref with no Store sends an empty name, and + Core takes the Store of History. """ - if input.operation_name not in ("AppendRecords", "CloseStream"): + if input.operation_name not in ("AppendRecords", "CloseStream", "ReadRecords"): return None payload_converter, _ = instance._system_nexus_converters(input) request = payload_converter._to_transfer_value(input.input) + if isinstance(request, temporalio.api.streamservice.v1.ReadRecordsRequest): + # A replay of a read whose marker holds its Records reads no Store. + return _start_direct_read( + instance, input, request, request.ref.store, records_in_history=True + ) return _start_direct_call(instance, input, request, request.ref.store) -def _read_command( # pyright: ignore[reportUnusedFunction] - reading: temporalio.nexus.system._support.StreamReading[Any], +# A client's read gives at most this many Records in a page, and a `max_records` of zero +# or less gives this many. A read of Workflow code gives the same pages. +_MAX_READ_RECORDS = 1000 +# The longest `wait` of a client's read. A longer one waits this long. +_MAX_READ_WAIT = timedelta(seconds=60) + + +@dataclasses.dataclass(frozen=True) +class _ReadKey: + """What a ``read`` call of Workflow code must match to take an open read.""" + + store: str + stream_id: str + offset: bytes + """The offset that the next page starts after. Empty is the start of the Stream.""" + wait: timedelta | None + + +class _DirectRead: + """One Core read of a Stream, which serves the ``read`` calls of Workflow code. + + Core gives its Records in batches. Each ``read`` call takes one page of them, and the + read then waits for the next call under the offset of that page. + """ + + def __init__(self, seq: int) -> None: + self.seq = seq + self.records: collections.deque[temporalio.api.stream.v1.StreamRecord] = ( + collections.deque() + ) + # Whether Core still reads, with no end and no close. + self.running = True + self.done = False + self.error: BaseException | None = None + # A job and an end wake the call that waits, which is at most one. + self._waiters: list[asyncio.Future[None]] = [] + + def kept(self) -> bool: + """Whether the read can serve a later call.""" + # A failure that no call raised yet stays, so the next call raises it. + return self.running or bool(self.records) or self.error is not None + + def take( + self, + records: Iterable[temporalio.api.stream.v1.StreamRecord], + *, + end: bool, + done: bool, + error: BaseException | None, + ) -> None: + """Keep the Records and the end of one job of Core.""" + self.records.extend(records) + if end: + self.running = False + self.done = done + self.error = error + self._wake() + + def close(self) -> None: + """Mark the read as closed in Core. Its buffered Records stay.""" + self.running = False + self._wake() + + def _wake(self) -> None: + for waiter in self._waiters: + if not waiter.done(): + waiter.set_result(None) + + async def page( + self, max_records: int, max_bytes: int | None + ) -> tuple[list[temporalio.api.stream.v1.StreamRecord], bool]: + """Give the next page and whether the Stream is done, and wait when none is buffered. + + Raises: + temporalio.exceptions.StreamError: The read failed. + """ + while not self.records and self.running: + waiter: asyncio.Future[None] = asyncio.get_running_loop().create_future() + self._waiters.append(waiter) + try: + await waiter + finally: + self._waiters.remove(waiter) + if not self.records and self.error is not None: + error, self.error = self.error, None + raise error + page: list[temporalio.api.stream.v1.StreamRecord] = [] + size = 0 + while self.records and len(page) < max_records: + record_size = self.records[0].ByteSize() + if page and max_bytes is not None and size + record_size > max_bytes: + break + page.append(self.records.popleft()) + size += record_size + return page, self.done and not self.records + + +def _read_limits( + request: temporalio.api.streamservice.v1.ReadRecordsRequest, +) -> tuple[int, timedelta | None, int | None]: + """Give the ``max_records``, ``wait`` and ``max_bytes`` of a client's read. + + Raises: + temporalio.exceptions.StreamError: ``max_bytes`` is negative. + """ + max_records = request.max_records + if max_records <= 0 or max_records > _MAX_READ_RECORDS: + max_records = _MAX_READ_RECORDS + wait: timedelta | None = None + if request.HasField("wait") and request.wait.ToTimedelta() > timedelta(): + wait = min(request.wait.ToTimedelta(), _MAX_READ_WAIT) + if request.max_bytes < 0: + raise temporalio.exceptions.StreamError( + "max_bytes must be 0 or more", + reason=temporalio.api.enums.v1.StreamErrorReason.STREAM_ERROR_REASON_INVALID, + ) + return max_records, wait, request.max_bytes or None + + +def _start_direct_read( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: temporalio.api.streamservice.v1.ReadRecordsRequest, store_name: str, *, records_in_history: bool, -) -> temporalio.bridge.proto.workflow_commands.StartStreamRead: - """Build the start of a Direct read on the Store ``store_name``, with no ``seq``.""" - options = reading._options - start = temporalio.bridge.proto.workflow_commands.StartStreamRead( - stream=temporalio.api.stream.v1.StreamRef( - id=reading._stream.ref.id, store=store_name - ), - from_offset=options.from_offset or b"", - records_in_history=records_in_history, +) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Answer one ``read`` call of Workflow code from a Core read of the Stream.""" + instance._assert_not_read_only("start stream read") + payload_converter, _ = instance._system_nexus_converters(input) + max_records, wait, max_bytes = _read_limits(request) + key = _ReadKey( + store=store_name, + stream_id=request.ref.id, + offset=request.from_offset, + wait=wait, + ) + read = instance._take_stream_read(key) + if read is None: + start = temporalio.bridge.proto.workflow_commands.StartStreamRead( + stream=temporalio.api.stream.v1.StreamRef( + id=request.ref.id, store=store_name + ), + from_offset=request.from_offset, + records_in_history=records_in_history, + ) + # The pages come from the calls, so the Core read has no cap and no timeout. + # With no `wait`, an idle timeout of zero gives what is in the Stream. + start.idle_timeout.SetInParent() + start.idle_timeout.FromTimedelta(wait or timedelta()) + read = instance._start_stream_read(start) + + async def result() -> OutputT: + try: + records, done = await read.page(max_records, max_bytes) + except BaseException: + instance._park_stream_read(read, key) + raise + last_offset = records[-1].offset if records else request.from_offset + instance._park_stream_read(read, dataclasses.replace(key, offset=last_offset)) + return payload_converter._from_transfer_value( + temporalio.api.streamservice.v1.ReadRecordsResponse( + records=records, last_offset=last_offset, done=done + ), + input.output_type, + ) + + return _StreamStoreOperationHandle( + instance.create_task(result(), name=f"stream store: {input.operation_name}") ) - if options.timeout is not None: - start.timeout.FromTimedelta(options.timeout) - if options.idle_timeout is not None: - # Core reads an unset idle timeout as no bound and a zero as no wait, so only a - # value that the code gave goes in the command. - start.idle_timeout.FromTimedelta(options.idle_timeout) - if options.max_records is not None: - start.max_records = options.max_records - max_bytes = options.max_bytes - if max_bytes is None and records_in_history: - max_bytes = temporalio.common.DEFAULT_STREAM_MAX_BYTES - if max_bytes is not None: - start.max_bytes = max_bytes - return start def _bridge_read_ahead_bytes(config: StreamConfig | None) -> int | None: # pyright: ignore[reportUnusedFunction] @@ -263,23 +411,6 @@ def from_config(cls, config: StreamConfig | None) -> _WorkflowStreamStore | None """Build it from the Stream config of a client, or give ``None`` when it has none.""" return cls(config=config) if config is not None else None - def read_command( - self, reading: temporalio.nexus.system._support.StreamReading[Any] - ) -> temporalio.bridge.proto.workflow_commands.StartStreamRead: - """Build the start of a Direct read, with no ``seq``.""" - # The choice runs in Workflow code, so a ref of a Store that the config doesn't - # have raises `StreamConfigError` here, with no I/O, and fails the Workflow Task. - store = self.config._store_for(reading._stream.ref, _workflow_target()) - if not isinstance(store, _BridgeStreamStore): - raise _no_direct_read_error(store) - return _read_command( - reading, - store.name, - records_in_history=( - self.config.workflow_read_replay_from is StreamReplaySource.HISTORY - ), - ) - def start_operation( self, instance: _WorkflowInstanceImpl, @@ -287,12 +418,6 @@ def start_operation( ) -> temporalio.workflow.NexusOperationHandle[OutputT]: """Answer a Stream call of Workflow code with the Store that the call goes to.""" stream_operation = _STREAM_OPERATIONS[input.operation_name] - # Workflow code can start any operation of the generated service through a Nexus - # client, also one that the Workflow `Stream` leaves out. - if stream_operation.workflow_refusal is not None: - raise temporalio.exceptions.ApplicationError( - stream_operation.workflow_refusal, non_retryable=True - ) payload_converter, _ = instance._system_nexus_converters(input) request = payload_converter._to_transfer_value(input.input) ref = _request_ref(input.input) @@ -300,6 +425,19 @@ def start_operation( # have raises `StreamConfigError` here, with no I/O, and fails the Workflow Task. store = self.config._store_for(ref, _workflow_target()) options = self.config.workflow_call_options + if isinstance(request, temporalio.api.streamservice.v1.ReadRecordsRequest): + instance._assert_not_read_only("start stream read") + if not isinstance(store, _BridgeStreamStore): + raise _no_direct_read_error(store) + return _start_direct_read( + instance, + input, + request, + store.name, + records_in_history=( + self.config.workflow_read_replay_from is StreamReplaySource.HISTORY + ), + ) if isinstance(store, _BridgeStreamStore): # Core appends and closes for a Workflow Producer itself, with the Workflow # Task as its epoch. A Producer of another kind keeps the Local Activity path. diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 72d668245..521a5d00c 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -46,7 +46,6 @@ import temporalio.api.common.v1 import temporalio.api.enums.v1 import temporalio.api.sdk.v1 -import temporalio.api.stream.v1 import temporalio.bridge.proto.activity_result import temporalio.bridge.proto.child_workflow import temporalio.bridge.proto.common @@ -59,8 +58,6 @@ import temporalio.converter._serialization_context import temporalio.exceptions import temporalio.nexus.system -import temporalio.nexus.system._support -import temporalio.nexus.system.stream_service import temporalio.workflow from temporalio.client._stream_store import _STREAM_SERVICE from temporalio.converter import StorageDriverStoreContext, StorageDriverWorkflowInfo @@ -99,8 +96,9 @@ WorkflowOutboundInterceptor, ) from ._stream_store import ( + _DirectRead, _no_stream_config_error, - _read_command, + _ReadKey, _start_replayed_direct_call, _stream_error_from_core, _WorkflowStreamStore, @@ -275,6 +273,17 @@ def get_thread_id(self) -> int | None: return None +def _is_completion( + command: temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.WorkflowCommand, +) -> bool: + return ( + command.HasField("complete_workflow_execution") + or command.HasField("continue_as_new_workflow_execution") + or command.HasField("fail_workflow_execution") + or command.HasField("cancel_workflow_execution") + ) + + class UnsandboxedWorkflowRunner(WorkflowRunner): """Workflow runner that does not do any sandboxing.""" @@ -373,16 +382,18 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._pending_nexus_operations: dict[int, _NexusOperationHandle] = {} self._pending_external_signals: dict[int, tuple[asyncio.Future, str]] = {} self._pending_external_cancels: dict[int, tuple[asyncio.Future, str]] = {} - self._pending_stream_reads: dict[ - int, temporalio.nexus.system._support.StreamReading[Any] - ] = {} + # Core reads that run, by seq. + self._pending_stream_reads: dict[int, _DirectRead] = {} + # Reads that no call waits on, by what the next call must match. Each holds the + # Core read and the Records that it gave. + self._parked_stream_reads: dict[_ReadKey, list[_DirectRead]] = {} self._pending_stream_appends: dict[ int, asyncio.Future[temporalio.bridge.proto.workflow_activation.StreamAppended], ] = {} self._pending_stream_closes: dict[int, asyncio.Future[None]] = {} - # Reads that Workflow code closed. A job of one of them can still come in the - # activation that the close answers, and is dropped. + # Reads that the SDK closed. A job of one of them can still come in the activation + # that the close answers, and is dropped. self._closed_stream_reads: set[int] = set() # Keyed by type self._curr_seqs: dict[str, int] = {} @@ -597,6 +608,17 @@ def activate( self._run_once( check_conditions=index == 1 or index == 2, jobs=act.jobs ) + + # A query-only activation sends no command, and the end of the Workflow + # stops every read in Core. + if ( + not job_sets[3] + and not self._deleting + and not any( + map(_is_completion, self._current_completion.successful.commands) + ) + ): + self._close_parked_stream_reads() except Exception as err: # We want some errors during activation, like those that can happen # during payload conversion, to be able to fail the workflow not the @@ -661,17 +683,7 @@ def activate( ) self._current_completion.failed.failure.application_failure_info.SetInParent() - def is_completion( - command: temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.WorkflowCommand, - ): - return ( - command.HasField("complete_workflow_execution") - or command.HasField("continue_as_new_workflow_execution") - or command.HasField("fail_workflow_execution") - or command.HasField("cancel_workflow_execution") - ) - - if any(map(is_completion, self._current_completion.successful.commands)): + if any(map(_is_completion, self._current_completion.successful.commands)): self._warn_if_unfinished_handlers() return self._current_completion @@ -973,35 +985,22 @@ def _apply_remove_from_cache( def _apply_resolve_stream_read( self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamRead ) -> None: - reading = self._pending_stream_reads.get(job.seq) - if reading is None: + read = self._pending_stream_reads.get(job.seq) + if read is None: if job.seq in self._closed_stream_reads: return raise RuntimeError(f"Failed finding Stream read for sequence {job.seq}") - end: ( - tuple[ - temporalio.nexus.system._support.StreamReadStop | None, - bool, - BaseException | None, - ] - | None - ) = None - if job.HasField("end"): + error: BaseException | None = None + end = job.HasField("end") + if end: del self._pending_stream_reads[job.seq] - error: BaseException | None = None if job.end.HasField("failure"): error = _stream_error_from_core(job.end.failure) if error is None: error = self._context_free_failure_converter.from_failure( job.end.failure, self._context_free_payload_converter ) - stop = ( - temporalio.nexus.system._support.StreamReadStop(job.end.stop) - if job.end.stop - else None - ) - end = (stop, job.end.done, error) - reading._take(job.records, end) + read.take(job.records, end=end, done=job.end.done, error=error) def _apply_resolve_stream_append( self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamAppend @@ -1873,39 +1872,43 @@ async def workflow_start_child_workflow( ) ) - def workflow_start_stream_read( - self, reading: temporalio.nexus.system._support.StreamReading[Any] - ) -> None: - self._assert_not_read_only("start stream read") - store = self._stream_store - if store is not None: - start = store.read_command(reading) - elif self._is_replaying: - # A replay of a read whose marker holds its Records reads no Store, so it needs - # no Stream config. A ref that names no Store sends an empty name, and Core - # takes the Store of History. - start = _read_command( - reading, reading._stream.ref.store or "", records_in_history=True - ) - else: - raise _no_stream_config_error() + def _start_stream_read( + self, start: temporalio.bridge.proto.workflow_commands.StartStreamRead + ) -> _DirectRead: start.seq = self._next_seq("stream_read") self._add_command().start_stream_read.CopyFrom(start) - # Stream calls have no serialization context, as the system Nexus converters of - # the other Stream calls show. - payload_converter = temporalio.nexus.system._get_payload_converter( - self._context_free_payload_converter, self._context_free_failure_converter - ) - body_type: Any = reading._stream.body_type - record_type = temporalio.nexus.system.stream_service.StreamRecord[body_type] - - def decode( - record: temporalio.api.stream.v1.StreamRecord, - ) -> temporalio.nexus.system.stream_service.StreamRecord[Any]: - return payload_converter._from_transfer_value(record, record_type) + read = _DirectRead(start.seq) + self._pending_stream_reads[start.seq] = read + return read - reading._started(start.seq, decode) - self._pending_stream_reads[start.seq] = reading + def _take_stream_read(self, key: _ReadKey) -> _DirectRead | None: + reads = self._parked_stream_reads.get(key) + if not reads: + return None + read = reads.pop(0) + if not reads: + del self._parked_stream_reads[key] + return read + + def _park_stream_read(self, read: _DirectRead, key: _ReadKey) -> None: + if read.kept(): + self._parked_stream_reads.setdefault(key, []).append(read) + + def _close_parked_stream_reads(self) -> None: + # A Core read holds its Workflow Task open, so one that no call waits on closes + # before the activation completes. Its Records stay for a later call. + for key, reads in list(self._parked_stream_reads.items()): + for read in reads: + if read.running: + self._add_command().close_stream_read.seq = read.seq + self._closed_stream_reads.add(read.seq) + del self._pending_stream_reads[read.seq] + read.close() + kept = [read for read in reads if read.kept()] + if kept: + self._parked_stream_reads[key] = kept + else: + del self._parked_stream_reads[key] def _start_stream_append( self, append: temporalio.bridge.proto.workflow_commands.AppendToStream @@ -1929,22 +1932,6 @@ def _start_stream_close( self._pending_stream_closes[close.seq] = fut return fut - def workflow_close_stream_read( - self, - reading: temporalio.nexus.system._support.StreamReading[Any], - *, - send: bool, - ) -> None: - seq = reading._seq - if seq is None or seq not in self._pending_stream_reads: - return - # An eviction ends every read in Core, so it needs no close. - if send and not self._deleting: - self._assert_not_read_only("close stream read") - self._add_command().close_stream_read.seq = seq - self._closed_stream_reads.add(seq) - del self._pending_stream_reads[seq] - def workflow_start_local_activity( self, activity: Any, diff --git a/temporalio/workflow/__init__.py b/temporalio/workflow/__init__.py index 4a2eb9708..60b02cc54 100644 --- a/temporalio/workflow/__init__.py +++ b/temporalio/workflow/__init__.py @@ -12,9 +12,6 @@ ) # END GENERATED NEXUS SYSTEM EXPORTS -# isort: split -from temporalio.nexus.system._support import StreamReading, StreamReadStop - from ..types import ( AnyType, CallableAsyncNoParam, @@ -335,8 +332,6 @@ "ProtocolReturnType", "ReturnType", "SelfType", - "StreamReading", - "StreamReadStop", # BEGIN GENERATED NEXUS SYSTEM __ALL__ "create_stream", "get_stream", diff --git a/temporalio/workflow/_context.py b/temporalio/workflow/_context.py index a215813d3..a33759c37 100644 --- a/temporalio/workflow/_context.py +++ b/temporalio/workflow/_context.py @@ -500,12 +500,6 @@ async def workflow_start_nexus_operation( event_groups: Sequence[EventGroup] | None = None, ) -> NexusOperationHandle[OutputT]: ... - @abstractmethod - def workflow_start_stream_read(self, reading: Any) -> None: ... - - @abstractmethod - def workflow_close_stream_read(self, reading: Any, *, send: bool) -> None: ... - @abstractmethod def workflow_time_ns(self) -> int: ... diff --git a/tests/contrib/redis/test_streams.py b/tests/contrib/redis/test_streams.py index 6dd5c9325..742911199 100644 --- a/tests/contrib/redis/test_streams.py +++ b/tests/contrib/redis/test_streams.py @@ -198,7 +198,7 @@ async def append_in_two_steps(stream_id: str, seen_id: str) -> None: @workflow.defn -class IterWhileActivityRunsWorkflow: +class ReadWhileActivityRunsWorkflow: @workflow.run async def run(self, stream_id: str, seen_id: str) -> list[str]: writer = workflow.start_activity( @@ -209,35 +209,36 @@ async def run(self, stream_id: str, seen_id: str) -> list[str]: stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) seen = workflow.get_stream(StreamRef(id=seen_id), body_type=str) got: list[str] = [] - # The idle default of 1 minute is far longer than the pause between the two steps. - reading = stream.iter() - async for record in reading: - got.append(record.body) - if record.body == "b": - await seen.append("the first step") - assert reading.stop is not None - got.append(reading.stop.name) + # The wait is far longer than the pause between the two steps, so the loop ends + # only with the page that is done. + async for page in stream.iter_read(wait=timedelta(seconds=30)): + for record in page.records: + got.append(record.body) + if record.body == "b": + await seen.append("the first step") + if page.done: + got.append("done") await writer return got -async def test_workflow_iter_reads_an_activity_on_redis(client: Client): +async def test_workflow_read_of_an_activity_on_redis(client: Client): stream_client = _with_store(client, _store()) stream = await _create_stream(stream_client) seen = await _create_stream(stream_client) async with new_worker( - stream_client, IterWhileActivityRunsWorkflow, activities=[append_in_two_steps] + stream_client, ReadWhileActivityRunsWorkflow, activities=[append_in_two_steps] ) as worker: handle = await stream_client.start_workflow( - IterWhileActivityRunsWorkflow.run, + ReadWhileActivityRunsWorkflow.run, args=[stream.ref.id, seen.ref.id], id=f"workflow-{uuid.uuid4()}", task_queue=worker.task_queue, ) - assert await handle.result() == ["a", "b", "c", "d", "Done"] + assert await handle.result() == ["a", "b", "c", "d", "done"] # The default replay source keeps no Record in History, so the replay reads Redis. await Replayer( - workflows=[IterWhileActivityRunsWorkflow], + workflows=[ReadWhileActivityRunsWorkflow], stream_config=stream_client.config()["stream_config"], ).replay_workflow(await handle.fetch_history()) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 067fa8258..1792b4c4e 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -14,7 +14,11 @@ from temporalio import activity, workflow from temporalio.api.common.v1 import Payload from temporalio.api.enums.v1 import EventType, StreamErrorReason -from temporalio.bridge.proto.external_data import StreamCloseEntry, StreamMarkerData +from temporalio.bridge.proto.external_data import ( + StreamCloseEntry, + StreamMarkerData, + StreamReadEntry, +) from temporalio.client import ( Client, StreamConfig, @@ -41,7 +45,15 @@ ) from temporalio.runtime import MetricBuffer, Runtime, TelemetryConfig from temporalio.testing import MemoryStreamStore, WorkflowEnvironment -from temporalio.worker import Replayer, Worker +from temporalio.worker import ( + Interceptor, + Replayer, + StartNexusOperationInput, + Worker, + WorkflowInboundInterceptor, + WorkflowInterceptorClassInput, + WorkflowOutboundInterceptor, +) from tests.helpers import assert_eventually, assert_task_fail_eventually, new_worker from tests.helpers.streams import DictStreamStore from tests.test_converter import SimpleCodec @@ -137,7 +149,7 @@ async def _run( return handle, await handle.result() -# A generator over `iter` that reads the Stream of an Activity that still runs. +# A generator over `iter_read_items` that reads the Stream of an Activity that still runs. @dataclass @@ -169,8 +181,8 @@ async def stream_model_output(call: ModelCall) -> None: async def model_chunks(chunks_id: str) -> AsyncGenerator[str, None]: """Yield each chunk of the model's Stream, as the model stub of a library does.""" stream = workflow.get_stream(StreamRef(id=chunks_id), body_type=str) - # The idle default of 1 minute is far longer than a pause of the model, so one read waits. - async for record in stream.iter(): + # The wait is far longer than a pause of the model, so no page comes back empty. + async for record in stream.iter_read_items(wait=timedelta(seconds=30)): yield record.body @@ -191,7 +203,9 @@ async def run(self, call: ModelCall) -> str: return text -async def test_generator_over_iter_reads_a_running_activity(client: Client): +async def test_generator_over_iter_read_items_reads_a_running_activity( + client: Client, +): stream_client = _stream_client(client) chunks = await _filled_stream(stream_client, [], close=False) seen = await _filled_stream(stream_client, [], close=False) @@ -210,27 +224,28 @@ async def test_generator_over_iter_reads_a_running_activity(client: Client): assert max(_stream_markers_per_task(history)) == 1 -# One read and its stop. +# The pages of a read. @dataclass class ReadArgs: stream_id: str from_offset: str | None = None - timeout_ms: int | None = None - idle_timeout_ms: int | None = None + wait_ms: int | None = None max_records: int | None = None max_bytes: int | None = None store: str = "memory" + pages: int | None = None + """The most pages to read. ``None`` reads until the iterator ends.""" @dataclass class ReadResult: bodies: list[str] offsets: list[str] - last_offset: str | None - stop: str | None - done: bool + page_sizes: list[int] + last_offsets: list[str | None] + dones: list[bool] def _ms(value: int | None) -> timedelta | None: @@ -242,147 +257,180 @@ def _hex(value: bytes | None) -> str | None: @workflow.defn -class ReadOnceWorkflow: +class ReadPagesWorkflow: @workflow.run async def run(self, args: ReadArgs) -> ReadResult: - # The ref names its Store, so a replay with no Stream config knows it. + # The ref has its Store, so a replay with no Stream config knows it. ref = StreamRef[str](id=args.stream_id, store=args.store) stream = workflow.get_stream(ref, body_type=str) - reading = stream.iter( - timeout=_ms(args.timeout_ms), + result = ReadResult([], [], [], [], []) + async for page in stream.iter_read( from_offset=( bytes.fromhex(args.from_offset) if args.from_offset is not None else None ), - idle_timeout=_ms(args.idle_timeout_ms), max_records=args.max_records, + wait=_ms(args.wait_ms), max_bytes=args.max_bytes, - ) - assert reading.stop is None and not reading.done - records = [record async for record in reading] - return ReadResult( - bodies=[record.body for record in records], - offsets=[record.offset.hex() for record in records], - last_offset=_hex(reading.last_offset), - stop=reading.stop.name if reading.stop is not None else None, - done=reading.done, - ) + ): + result.bodies.extend(record.body for record in page.records) + result.offsets.extend(record.offset.hex() for record in page.records) + result.page_sizes.append(len(page.records)) + result.last_offsets.append(_hex(page.last_offset)) + result.dones.append(page.done) + if len(result.page_sizes) == args.pages: + break + return result -async def _read_once(client: Client, args: ReadArgs) -> ReadResult: - _, result = await _run(client, ReadOnceWorkflow, args) +async def _read_pages( + client: Client, args: ReadArgs, *, max_cached_workflows: int = 1000 +) -> ReadResult: + _, result = await _run( + client, ReadPagesWorkflow, args, max_cached_workflows=max_cached_workflows + ) return result -async def test_read_stops_at_done(client: Client): +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_closed_stream_ends_the_iterator_with_a_done_page( + client: Client, max_cached_workflows: int +): stream_client = _stream_client(client) ref = await _filled_stream(stream_client, ["a", "b", "c"]) - result = await _read_once(stream_client, ReadArgs(ref.id)) + result = await _read_pages( + stream_client, + ReadArgs(ref.id, wait_ms=30_000), + max_cached_workflows=max_cached_workflows, + ) assert result.bodies == ["a", "b", "c"] - assert (result.stop, result.done) == ("Done", True) - assert result.last_offset == result.offsets[-1] - - -async def test_read_stops_at_idle(client: Client): - stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, ["a", "b"], close=False) - result = await _read_once(stream_client, ReadArgs(ref.id, idle_timeout_ms=200)) - assert result.bodies == ["a", "b"] - assert (result.stop, result.done) == ("Idle", False) - assert result.last_offset == result.offsets[-1] + # The iterator ends after the page with `done`, also with a `wait`. + assert result.dones[-1] is True + assert not any(result.dones[:-1]) + assert result.last_offsets[-1] == result.offsets[-1] -async def test_read_stops_at_timeout(client: Client): +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_read_with_max_records_gives_pages_of_two_two_and_one( + client: Client, max_cached_workflows: int +): stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, ["a"], close=False) - result = await _read_once( - stream_client, ReadArgs(ref.id, timeout_ms=500, idle_timeout_ms=30_000) + ref = await _filled_stream(stream_client, ["a", "b", "c", "d", "e"]) + result = await _read_pages( + stream_client, + ReadArgs(ref.id, max_records=2), + max_cached_workflows=max_cached_workflows, ) - assert result.bodies == ["a"] - assert (result.stop, result.done) == ("Timeout", False) + assert result.bodies == ["a", "b", "c", "d", "e"] + assert result.page_sizes == [2, 2, 1] + assert result.dones == [False, False, True] + # Each page ends at its last Record, and the next one starts after it. + assert result.last_offsets == [ + result.offsets[1], + result.offsets[3], + result.offsets[4], + ] -async def test_read_past_its_timeout_gives_what_is_there(client: Client): - stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, ["a", "b"], close=False) - # The timeout passes before Core starts the read, which still looks at the Store once. - result = await _read_once( - stream_client, ReadArgs(ref.id, timeout_ms=1, idle_timeout_ms=30_000) - ) - assert result.bodies == ["a", "b"] - assert (result.stop, result.done) == ("Timeout", False) +@workflow.defn +class ReadOnePageWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str], wait_ms: int | None) -> list[Any]: + stream = workflow.get_stream(ref, body_type=str) + page = await stream.read(wait=_ms(wait_ms)) + return [[record.body for record in page.records], page.done, page.last_offset] -async def test_read_stops_at_max_records(client: Client): +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_read_with_wait_on_a_quiet_stream_gives_an_empty_page( + client: Client, max_cached_workflows: int +): stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, ["a", "b", "c", "d"]) - result = await _read_once(stream_client, ReadArgs(ref.id, max_records=3)) - assert result.bodies == ["a", "b", "c"] - assert (result.stop, result.done) == ("MaxRecords", False) - # The next read starts after the last Record that this one gave. - rest = await _read_once( - stream_client, ReadArgs(ref.id, from_offset=result.last_offset) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run( + stream_client, + ReadOnePageWorkflow, + ref, + 1000, + max_cached_workflows=max_cached_workflows, ) - assert rest.bodies == ["d"] - assert (rest.stop, rest.done) == ("Done", True) - - -async def test_read_that_takes_the_last_record_with_a_cap_is_done(client: Client): - stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, ["a", "b", "c"]) - # The cap takes the last Record of the closed Stream, so the stop is Done. - by_records = await _read_once(stream_client, ReadArgs(ref.id, max_records=3)) - assert by_records.bodies == ["a", "b", "c"] - assert (by_records.stop, by_records.done) == ("Done", True) - one = await _filled_stream(stream_client, ["a"]) - by_bytes = await _read_once(stream_client, ReadArgs(one.id, max_bytes=1)) - assert by_bytes.bodies == ["a"] - assert (by_bytes.stop, by_bytes.done) == ("Done", True) - # On an open Stream the same cap stops the read with its own stop. - open_ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) - capped = await _read_once(stream_client, ReadArgs(open_ref.id, max_records=3)) - assert (capped.stop, capped.done) == ("MaxRecords", False) + assert result == [[], False, None] + # The page comes only after the wait. + events = (await handle.fetch_history()).events + elapsed = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert elapsed >= timedelta(seconds=1) -async def test_read_stops_at_max_bytes(client: Client): +async def test_read_with_no_wait_on_an_empty_stream_gives_an_empty_page( + client: Client, +): stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, ["a", "b", "c"]) - # The first Record always comes, even over the cap. - result = await _read_once(stream_client, ReadArgs(ref.id, max_bytes=1)) - assert result.bodies == ["a"] - assert (result.stop, result.done) == ("MaxBytes", False) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run(stream_client, ReadOnePageWorkflow, ref, None) + assert result == [[], False, None] + events = (await handle.fetch_history()).events + # An empty Stream gives no reason to wait. + took = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert took < timedelta(seconds=5) -async def test_read_with_zero_idle_timeout_gives_what_is_there(client: Client): +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_iter_read_items_with_no_wait_gives_what_is_there_and_ends( + client: Client, max_cached_workflows: int +): stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, ["a", "b"], close=False) - handle, result = await _run( - stream_client, ReadOnceWorkflow, ReadArgs(ref.id, idle_timeout_ms=0) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + handle, bodies = await _run( + stream_client, + ReadItemsWorkflow, + ref, + None, + max_cached_workflows=max_cached_workflows, ) - assert result.bodies == ["a", "b"] - assert (result.stop, result.done) == ("Idle", False) + assert bodies == ["a", "b", "c"] events = (await handle.fetch_history()).events # The read doesn't wait for the Store, so the run ends long before the first heartbeat. took = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() assert took < timedelta(seconds=5) -async def test_read_with_zero_idle_timeout_ends_immediately_on_an_empty_stream( +@workflow.defn +class ReadItemsWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str], wait_ms: int | None) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + return [ + record.body async for record in stream.iter_read_items(wait=_ms(wait_ms)) + ] + + +async def test_page_that_takes_the_last_record_with_a_cap_is_done(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + # The cap takes the last Record of the closed Stream, so the page is done. + by_records = await _read_pages(stream_client, ReadArgs(ref.id, max_records=3)) + assert (by_records.page_sizes, by_records.dones) == ([3], [True]) + one = await _filled_stream(stream_client, ["a"]) + by_bytes = await _read_pages(stream_client, ReadArgs(one.id, max_bytes=1)) + assert (by_bytes.page_sizes, by_bytes.dones) == ([1], [True]) + # On an open Stream the same page isn't done. + open_ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + capped = await _read_pages( + stream_client, ReadArgs(open_ref.id, max_records=3, pages=1) + ) + assert (capped.page_sizes, capped.dones) == ([3], [False]) + + +async def test_page_with_max_bytes_gives_the_first_record_over_the_cap( client: Client, ): stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, [], close=False) - handle, result = await _run( - stream_client, ReadOnceWorkflow, ReadArgs(ref.id, idle_timeout_ms=0) - ) - assert result.bodies == [] - assert (result.stop, result.done) == ("Idle", False) - assert result.last_offset is None - events = (await handle.fetch_history()).events - # An empty Stream gives no reason to wait either. - took = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() - assert took < timedelta(seconds=5) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + result = await _read_pages(stream_client, ReadArgs(ref.id, max_bytes=1)) + # The first Record of each page always comes, even over the cap. + assert result.page_sizes == [1, 1, 1] + assert result.dones == [False, False, True] @activity.defn @@ -397,69 +445,191 @@ async def append_after_a_quiet_gap(stream_id: str) -> None: @workflow.defn class QuietGapWorkflow: @workflow.run - async def run(self, stream_id: str) -> list[Any]: + async def run(self, stream_id: str) -> list[str]: writer = workflow.start_activity( append_after_a_quiet_gap, stream_id, start_to_close_timeout=timedelta(seconds=30), ) - reading = workflow.get_stream(StreamRef(id=stream_id), body_type=str).iter() - got = [record.body async for record in reading] + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [ + record.body + async for record in stream.iter_read_items(wait=timedelta(seconds=10)) + ] await writer - return [got, reading.stop and reading.stop.name, reading.done] + return got -async def test_plain_iter_reads_across_a_quiet_gap(client: Client): +async def test_iter_read_items_with_a_wait_reads_across_a_quiet_gap(client: Client): stream_client = _stream_client(client) ref = await _filled_stream(stream_client, [], close=False) - _, result = await _run( + handle, result = await _run( stream_client, QuietGapWorkflow, ref.id, activities=[append_after_a_quiet_gap], ) - # One read gets both Records across the gap of 2 s, with no Idle stop in between. - assert result == [["a", "b"], "Done", True] + assert result == ["a", "b"] + # The gap of 2 s is shorter than the wait, so one Core read gets both Records. + assert len(_read_seqs(await handle.fetch_history())) == 1 + + +# A loop with no await in its body, and one that awaits for each Record. + + +def _read_entries(history: WorkflowHistory) -> list[list[StreamReadEntry]]: + """Give the read entries of the `core_stream` marker of each Workflow Task.""" + entries: list[list[StreamReadEntry]] = [] + for event in history.events: + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + entries.append(list(StreamMarkerData.FromString(payload.data).reads)) + return entries + + +def _read_seqs(history: WorkflowHistory) -> set[int]: + """Give the seq of each Core read that History has.""" + return {entry.seq for task in _read_entries(history) for entry in task} -# What `iter` checks and when the read starts. +@activity.defn +async def append_five_slowly(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + for index in range(5): + await asyncio.sleep(0.3) + await stream.append(f"record-{index}") + await stream.close() @workflow.defn -class IterArgumentsWorkflow: +class LoopWithNoAwaitWorkflow: @workflow.run - async def run(self, ref: StreamRef[str]) -> list[str]: + async def run(self, stream_id: str) -> list[str]: + writer = workflow.start_activity( + append_five_slowly, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [] + async for record in stream.iter_read_items(wait=timedelta(seconds=10)): + got.append(record.body) + await writer + return got + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_loop_with_no_await_keeps_one_core_read( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, got = await _run( + stream_client, + LoopWithNoAwaitWorkflow, + ref.id, + activities=[append_five_slowly], + max_cached_workflows=max_cached_workflows, + ) + assert got == [f"record-{index}" for index in range(5)] + entries = _read_entries(await handle.fetch_history()) + # The loop asks for the next page in the activation of the last one, so one Core + # read gives every page. + assert len({entry.seq for task in entries for entry in task}) == 1 + assert any(len(task) == 1 and len(task[0].page_counts) > 1 for task in entries) + + +@workflow.defn +class LoopWithAppendWorkflow: + @workflow.run + async def run(self, stream_id: str, out_id: str) -> list[str]: + writer = workflow.start_activity( + append_five_slowly, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + out = workflow.get_stream(StreamRef(id=out_id), body_type=str) + got = [] + async for record in stream.iter_read_items(wait=timedelta(seconds=10)): + got.append(record.body) + await out.append(record.body) + await writer + return got + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_loop_that_awaits_an_append_closes_and_starts_the_read( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + out_ref = await _filled_stream(stream_client, [], close=False) + handle, got = await _run( + stream_client, + LoopWithAppendWorkflow, + ref.id, + out_ref.id, + activities=[append_five_slowly], + max_cached_workflows=max_cached_workflows, + ) + bodies = [f"record-{index}" for index in range(5)] + assert got == bodies + # Each Record went to the other Stream once and in order, whatever the replays. + assert [record.body for record in await _read_all(stream_client, out_ref)] == bodies + # The append ends the activation with no call waiting on the read, so the read + # closes, and the next page starts another one. + assert len(_read_seqs(await handle.fetch_history())) > 1 + + +# What `read` checks, and where it can't run. + + +@workflow.defn +class ReadLimitsWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[Any]: stream = workflow.get_stream(ref, body_type=str) - errors: list[str] = [] + results: list[Any] = [] cases: list[dict[str, Any]] = [ - {"timeout": timedelta()}, - {"timeout": timedelta(seconds=-1)}, - {"idle_timeout": timedelta(seconds=-1)}, - {"max_records": 0}, {"max_bytes": -1}, + {"max_records": 0}, + {"max_records": -1}, + {"max_records": 5000}, + {"wait": timedelta(seconds=-1)}, + {"wait": timedelta(seconds=120)}, ] for arguments in cases: try: - stream.iter(**arguments) - except ValueError as err: - errors.append(str(err)) - # `iter` sends nothing, so a read that the code never steps writes no marker. - stream.iter(idle_timeout=timedelta()) - return errors + page = await stream.read(**arguments) + results.append(len(page.records)) + except StreamError as err: + results.append([StreamErrorReason.Name(err.reason), err.message]) + return results -async def test_iter_checks_its_arguments_and_sends_nothing(client: Client): +async def test_read_takes_the_limits_of_a_client_read(client: Client): stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, ["a"]) - handle, errors = await _run(stream_client, IterArgumentsWorkflow, ref) - assert errors == [ - "timeout must be positive", - "timeout must be positive", - "idle_timeout must not be negative", - "max_records must be positive", - "max_bytes must be positive", - ] - assert _markers(await handle.fetch_history()) == [] + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + _, results = await _run(stream_client, ReadLimitsWorkflow, ref) + stream = stream_client.get_stream(ref, body_type=str) + expected: list[Any] = [] + for arguments in [ + {"max_bytes": -1}, + {"max_records": 0}, + {"max_records": -1}, + {"max_records": 5000}, + {"wait": timedelta(seconds=-1)}, + {"wait": timedelta(seconds=120)}, + ]: + try: + page = await stream.read(**arguments) + expected.append(len(page.records)) + except StreamError as err: + expected.append([StreamErrorReason.Name(err.reason), err.message]) + assert results == expected + assert results[0] == ["STREAM_ERROR_REASON_INVALID", expected[0][1]] @workflow.defn @@ -475,8 +645,7 @@ async def run(self, ref: StreamRef[str]) -> None: @workflow.query def start_read(self) -> str: assert self.ref is not None - reading = workflow.get_stream(self.ref, body_type=str).iter() - step = reading.__anext__() + step = workflow.get_stream(self.ref, body_type=str).read() try: step.send(None) except Exception as err: @@ -502,49 +671,38 @@ async def test_read_in_a_query_raises_the_read_only_error(client: Client): await handle.terminate() -# A loop of reads from `last_offset`. +# Reads that resume from an offset. @workflow.defn class LoopReadWorkflow: @workflow.run - async def run(self, ref: StreamRef[str]) -> list[Any]: + async def run(self, ref: StreamRef[str]) -> list[str]: stream = workflow.get_stream(ref, body_type=str) got: list[str] = [] - # The first read leaves the loop with `break` and no close. - first = stream.iter() - async for record in first: + # The first loop leaves with `break`, and its next request resumes there. + items = stream.iter_read_items(max_records=4) + async for record in items: got.append(record.body) if len(got) == 3: break - # The second read closes in the middle. - second = stream.iter(from_offset=first.last_offset) - async for record in second: - got.append(record.body) - if len(got) == 5: - await second.aclose() - await second.aclose() - after_close = [record.body async for record in second] - closed = [second.stop, second.done, after_close] - # The last reads go on from there to the end. - last = second.last_offset + assert items.next_request is not None + last = items.next_request.from_offset + # The last reads go on from there to the end, two Records at a time. while True: - reading = stream.iter(from_offset=last, max_records=2) - async for record in reading: - got.append(record.body) - last = reading.last_offset - if reading.done: - break - return [got, closed] + page = await stream.read(from_offset=last, max_records=2) + got.extend(record.body for record in page.records) + last = page.last_offset + if page.done: + return got -async def test_loop_from_last_offset_gets_each_record_once(client: Client): +async def test_reads_from_an_offset_get_each_record_once(client: Client): stream_client = _stream_client(client) bodies = [f"record-{index}" for index in range(10)] ref = await _filled_stream(stream_client, bodies) - _, (got, closed) = await _run(stream_client, LoopReadWorkflow, ref) + _, got = await _run(stream_client, LoopReadWorkflow, ref) assert got == bodies - assert closed == [None, False, []] # The marker in History. @@ -555,7 +713,7 @@ async def _history_of_a_read( ) -> tuple[WorkflowHistory, StreamRef[str], Client]: stream_client = _stream_client(client, replay_from=replay_from) ref = await _filled_stream(stream_client, [f"{_MARK}-1", f"{_MARK}-2"]) - handle, result = await _run(stream_client, ReadOnceWorkflow, ReadArgs(ref.id)) + handle, result = await _run(stream_client, ReadPagesWorkflow, ReadArgs(ref.id)) assert result.bodies == [f"{_MARK}-1", f"{_MARK}-2"] return await handle.fetch_history(), ref, stream_client @@ -589,7 +747,7 @@ async def test_replay_of_store_marker_reads_the_store(client: Client): client, StreamReplaySource.STORE ) config = stream_client.config()["stream_config"] - await Replayer(workflows=[ReadOnceWorkflow], stream_config=config).replay_workflow( + await Replayer(workflows=[ReadPagesWorkflow], stream_config=config).replay_workflow( history ) @@ -598,7 +756,7 @@ async def _failed_replay( history: WorkflowHistory, ref: StreamRef[str], config: StreamConfig | None ) -> str: """Replay with and without a raise, and give the text of the failure.""" - replayer = Replayer(workflows=[ReadOnceWorkflow], stream_config=config) + replayer = Replayer(workflows=[ReadPagesWorkflow], stream_config=config) with pytest.raises(RuntimeError) as err: await replayer.replay_workflow(history) result = await replayer.replay_workflow(history, raise_on_replay_failure=False) @@ -626,7 +784,7 @@ async def test_replay_of_store_marker_fails_when_records_are_gone(client: Client async def test_replay_of_history_marker_needs_no_stream_config(client: Client): history, _, _ = await _history_of_a_read(client, StreamReplaySource.HISTORY) - await Replayer(workflows=[ReadOnceWorkflow]).replay_workflow(history) + await Replayer(workflows=[ReadPagesWorkflow]).replay_workflow(history) # Two reads at once, on a Worker with no cache and on a cached one. @@ -664,16 +822,16 @@ async def _read(self, stream_id: str, out: Any) -> list[str]: last = None while True: self.reads += 1 - reading = stream.iter(from_offset=last, max_records=2) - async for record in reading: + page = await stream.read(from_offset=last, max_records=2) + for record in page.records: tagged = await workflow.execute_local_activity( tag, record.body, start_to_close_timeout=timedelta(seconds=10) ) await out.append(tagged) self.count += 1 got.append(tagged) - last = reading.last_offset - if reading.done: + last = page.last_offset + if page.done: return got # A timer ends the Workflow Task, so a Worker with no cache replays each read. await workflow.sleep(0.05) @@ -817,7 +975,7 @@ async def run(self, ref: StreamRef[str], out_ref: StreamRef[str]) -> list[str]: stream = workflow.get_stream(ref, body_type=str) out = workflow.get_stream(out_ref, body_type=str) got: list[str] = [] - async for record in stream.iter(): + async for record in stream.iter_read_items(): got.append(record.body) if record.body == "record-2": await out.append("seen record-2") @@ -851,34 +1009,44 @@ async def test_failed_task_in_a_read_gets_each_record_once(client: Client): assert epochs[0] < epochs[1] -# A Signal while a read runs. +# A Signal while a read waits. @workflow.defn -class SignalCloseWorkflow: +class SignalStopsReadWorkflow: def __init__(self) -> None: - self.reading: Any = None self.signaled = False @workflow.run async def run(self, ref: StreamRef[str]) -> list[Any]: stream = workflow.get_stream(ref, body_type=str) - self.reading = stream.iter(idle_timeout=timedelta(seconds=60)) - got = [record.body async for record in self.reading] - return [got, self.signaled, self.reading.stop] + got: list[str] = [] + + async def read_all() -> None: + async for record in stream.iter_read_items(wait=timedelta(seconds=60)): + got.append(record.body) + + reader = asyncio.create_task(read_all()) + await workflow.wait_condition(lambda: self.signaled) + # The cancel leaves the read with no call that waits on it, so it closes. + reader.cancel() + try: + await reader + except asyncio.CancelledError: + pass + return [got, self.signaled] @workflow.signal - async def stop(self) -> None: + def stop(self) -> None: self.signaled = True - await self.reading.aclose() -async def test_signal_reaches_a_running_read_and_closes_it(client: Client): +async def test_signal_reaches_a_waiting_read(client: Client): stream_client = _stream_client(client) ref = await _filled_stream(stream_client, ["a"], close=False) - async with new_worker(stream_client, SignalCloseWorkflow) as worker: + async with new_worker(stream_client, SignalStopsReadWorkflow) as worker: handle = await stream_client.start_workflow( - SignalCloseWorkflow.run, + SignalStopsReadWorkflow.run, ref, id=f"workflow-{uuid.uuid4()}", task_queue=worker.task_queue, @@ -886,12 +1054,16 @@ async def test_signal_reaches_a_running_read_and_closes_it(client: Client): task_timeout=timedelta(seconds=2), ) await asyncio.sleep(0.5) - await handle.signal(SignalCloseWorkflow.stop) + await handle.signal(SignalStopsReadWorkflow.stop) result = await asyncio.wait_for(handle.result(), timeout=30) - assert result == [["a"], True, None] + assert result == [["a"], True] + await Replayer( + workflows=[SignalStopsReadWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) -# A second read after a `break`, and the ends of a run with a read open. +# A second loop after a `break`, and the ends of a run with a read open. @activity.defn @@ -906,44 +1078,36 @@ class BreakThenReadAgainWorkflow: @workflow.run async def run(self, stream_id: str) -> list[Any]: stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) - first = stream.iter(idle_timeout=timedelta(seconds=30)) + first = stream.iter_read_items(wait=timedelta(seconds=30)) got_first: list[str] = [] async for record in first: got_first.append(record.body) if len(got_first) == 2: break - # The first read runs on in Core while the second reads the same Stream. - second = stream.iter( - from_offset=first.last_offset, idle_timeout=timedelta(seconds=30) + assert first.next_request is not None + # The second loop resumes after the last Record that the first one gave. + second = stream.iter_read( + from_offset=first.next_request.from_offset, wait=timedelta(seconds=30) ) writer = workflow.start_activity( append_and_close, stream_id, start_to_close_timeout=timedelta(seconds=30) ) - got_second = [record.body async for record in second] + got_second: list[str] = [] + done = False + async for page in second: + got_second.extend(record.body for record in page.records) + done = page.done await writer - rest_of_first = [record.body async for record in first] - return [ - got_first, - got_second, - second.stop and second.stop.name, - rest_of_first, - first.stop and first.stop.name, - ] + return [got_first, got_second, done] -async def test_second_read_after_a_break_gets_every_record(client: Client): +async def test_second_loop_after_a_break_gets_every_record(client: Client): stream_client = _stream_client(client) ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) handle, result = await _run( stream_client, BreakThenReadAgainWorkflow, ref.id, activities=[append_and_close] ) - assert result == [ - ["a", "b"], - ["c", "x", "y"], - "Done", - ["c", "x", "y"], - "Done", - ] + assert result == [["a", "b"], ["c", "x", "y"], True] await Replayer( workflows=[BreakThenReadAgainWorkflow], stream_config=stream_client.config()["stream_config"], @@ -955,14 +1119,11 @@ class CompleteWithOpenReadsWorkflow: @workflow.run async def run(self, stream_id: str) -> str: stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) - first = stream.iter(idle_timeout=timedelta(seconds=60)) - async for _ in first: - break - # A second read waits in `__anext__` in its own task while the Workflow returns. - second = stream.iter( - from_offset=first.last_offset, idle_timeout=timedelta(seconds=60) + first = await stream.read(wait=timedelta(seconds=60)) + # A second read waits in its own task while the Workflow returns. + waiting = asyncio.create_task( + stream.read(from_offset=first.last_offset, wait=timedelta(seconds=60)) ) - waiting = asyncio.create_task(second.__anext__()) await asyncio.sleep(0) assert not waiting.done() return "completed" @@ -990,15 +1151,13 @@ class CancelDuringReadWorkflow: @workflow.run async def run(self, stream_id: str) -> list[Any]: stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) - reading = stream.iter(idle_timeout=timedelta(seconds=60)) got: list[str] = [] try: - async for record in reading: + async for record in stream.iter_read_items(wait=timedelta(seconds=60)): got.append(record.body) except asyncio.CancelledError: - await reading.aclose() - return [got, "cancelled", reading.stop] - return [got, "ended", reading.stop] + return [got, "cancelled"] + return [got, "ended"] async def test_cancel_reaches_a_waiting_read(client: Client): @@ -1016,7 +1175,7 @@ async def test_cancel_reaches_a_waiting_read(client: Client): await asyncio.sleep(0.5) await handle.cancel() result = await asyncio.wait_for(handle.result(), timeout=30) - assert result == [["a"], "cancelled", None] + assert result == [["a"], "cancelled"] @workflow.defn @@ -1025,15 +1184,12 @@ class ContinueWithOpenReadWorkflow: async def run(self, stream_id: str, from_offset: str | None) -> list[str]: stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) if from_offset is None: - first = stream.iter(idle_timeout=timedelta(seconds=60)) - async for _ in first: - break + # The Core read of this page is still open when the run continues. + first = await stream.read(max_records=1, wait=timedelta(seconds=60)) assert first.last_offset is not None workflow.continue_as_new(args=[stream_id, first.last_offset.hex()]) - reading = stream.iter( - from_offset=bytes.fromhex(from_offset), idle_timeout=timedelta() - ) - return [record.body async for record in reading] + items = stream.iter_read_items(from_offset=bytes.fromhex(from_offset)) + return [record.body async for record in items] async def test_continue_as_new_with_an_open_read(client: Client): @@ -1079,7 +1235,7 @@ async def test_python_store_refuses_a_read(client: Client): stream_client = _stream_client(client, DictStreamStore()) ref = await _filled_stream(stream_client, ["a"]) with pytest.raises(WorkflowFailureError) as err: - await _run(stream_client, ReadOnceWorkflow, ReadArgs(ref.id, store="dict")) + await _run(stream_client, ReadPagesWorkflow, ReadArgs(ref.id, store="dict")) cause = err.value.cause assert isinstance(cause, ApplicationError) assert cause.type == "StreamConfigError" @@ -1100,7 +1256,7 @@ class AppendThenReadWorkflow: async def run(self, ref: StreamRef[str]) -> list[str]: stream = workflow.get_stream(ref, body_type=str) await stream.append("a", "b") - return [record.body async for record in stream.iter(idle_timeout=timedelta())] + return [record.body async for record in stream.iter_read_items()] async def test_payload_codec_covers_direct_append_and_read(client: Client): @@ -1170,7 +1326,7 @@ async def run(self, burst: Burst) -> int: ) stream = workflow.get_stream(StreamRef(id=burst.stream_id), body_type=str) count = 0 - async for _ in stream.iter(): + async for _ in stream.iter_read_items(wait=timedelta(seconds=30)): count += 1 await writer return count @@ -1200,72 +1356,56 @@ async def test_measure_read_of_200_records_over_2_seconds( ) -# Two tasks that share one read. +# Two loops from one offset, and calls on a read that fails. @workflow.defn -class SharedReadWorkflow: +class TwoLoopsWorkflow: @workflow.run - async def run(self, ref: StreamRef[str]) -> list[Any]: - reading = workflow.get_stream(ref, body_type=str).iter() - got: list[str] = [] + async def run(self, ref: StreamRef[str]) -> list[list[str]]: + stream = workflow.get_stream(ref, body_type=str) - async def drain() -> int: - count = 0 - async for record in reading: - got.append(record.body) - count += 1 - return count + async def drain() -> list[str]: + return [record.body async for record in stream.iter_read_items()] - # Both tasks wait in `__anext__` before the first page comes. + # Both loops ask for the first page before either gets it, so each one takes a + # read of its own. first = asyncio.create_task(drain()) second = asyncio.create_task(drain()) - try: - await workflow.wait_condition( - lambda: first.done() and second.done(), timeout=timedelta(seconds=5) - ) - except asyncio.TimeoutError: - first.cancel() - second.cancel() - return ["stuck", got] - assert reading.stop is not None - return [sorted(got), first.result() + second.result(), reading.stop.name] + return [await first, await second] -async def test_two_tasks_share_one_read(client: Client): +async def test_two_loops_from_one_offset_each_get_every_record(client: Client): stream_client = _stream_client(client) ref = await _filled_stream(stream_client, ["a", "b", "c"]) - _, result = await _run(stream_client, SharedReadWorkflow, ref) - # Each Record goes to one of the tasks, and both end when the read ends. - assert result == [["a", "b", "c"], 3, "Done"] + handle, result = await _run(stream_client, TwoLoopsWorkflow, ref) + assert result == [["a", "b", "c"], ["a", "b", "c"]] + assert len(_read_seqs(await handle.fetch_history())) == 2 @workflow.defn -class SharedFailedReadWorkflow: +class FailedReadsWorkflow: @workflow.run async def run(self, ref: StreamRef[str]) -> list[str]: - reading = workflow.get_stream(ref, body_type=str).iter() + stream = workflow.get_stream(ref, body_type=str) - async def step() -> str: + async def read() -> str: try: - await reading.__anext__() + await stream.read(wait=timedelta(seconds=10)) except StreamError as err: return StreamErrorReason.Name(err.reason) - except StopAsyncIteration: - return "end" - return "record" + return "page" - # Both tasks wait in `__anext__` before the failure comes. - first = asyncio.create_task(step()) - second = asyncio.create_task(step()) - return [await first, await second, await step()] + # Two calls wait before the failure comes, and a later call fails too. + first = asyncio.create_task(read()) + second = asyncio.create_task(read()) + return [await first, await second, await read()] -async def test_two_tasks_share_a_failed_read(client: Client): +async def test_each_read_of_a_missing_stream_raises(client: Client): stream_client = _stream_client(client) ref: StreamRef[str] = StreamRef(id=f"stream-{uuid.uuid4()}") - _, result = await _run(stream_client, SharedFailedReadWorkflow, ref) - # No task takes the failure for a clean end, and a later step raises it again. + _, result = await _run(stream_client, FailedReadsWorkflow, ref) assert result == ["STREAM_ERROR_REASON_NOT_FOUND"] * 3 @@ -1314,7 +1454,7 @@ class AppendThenReadAllWorkflow: async def run(self, ref: StreamRef[str]) -> list[str]: stream = workflow.get_stream(ref, body_type=str) await stream.append("from-workflow") - return [record.body async for record in stream.iter(idle_timeout=timedelta())] + return [record.body async for record in stream.iter_read_items()] @pytest.mark.parametrize("store_kind", ["memory", "dict"]) @@ -1360,7 +1500,7 @@ class UnnamedRefReadWorkflow: @workflow.run async def run(self, stream_id: str) -> list[str]: stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) - return [record.body async for record in stream.iter()] + return [record.body async for record in stream.iter_read_items()] async def _unnamed_read_history( @@ -1419,35 +1559,27 @@ class BadBodyWorkflow: @workflow.run async def run(self, stream_id: str) -> list[Any]: stream = workflow.get_stream(StreamRef(id=stream_id), body_type=Point) - reading = stream.iter() - events: list[Any] = [] - while True: + first = await stream.read(max_records=1) + events: list[Any] = [["got", first.records[0].body.x]] + # The page of the bad body raises, and a read from the same offset gets it again. + for _ in range(2): try: - record = await reading.__anext__() - events.append(["got", record.body.x]) - except StopAsyncIteration: - break + await stream.read(from_offset=first.last_offset, max_records=1) + events.append(["page", "decoded"]) except Exception as err: - events.append(["error", type(err).__name__]) - break - # A new read from `last_offset` gets the Record that failed again. - again = stream.iter(from_offset=reading.last_offset) - try: - await again.__anext__() - events.append(["again", "decoded"]) - except Exception as err: - events.append(["again", type(err).__name__]) - await again.aclose() + events.append(["page", type(err).__name__]) return events -async def test_last_offset_stays_before_a_body_that_fails_to_decode(client: Client): +async def test_page_with_a_body_that_fails_to_decode_raises_each_time( + client: Client, +): stream_client = _stream_client(client) stream = await stream_client.create_stream(id=f"s-{uuid.uuid4()}", body_type=object) await stream.append({"x": 1}, "not a point", {"x": 3}) await stream.close() _, events = await _run(stream_client, BadBodyWorkflow, stream.ref.id) - assert events == [["got", 1], ["error", "TypeError"], ["again", "TypeError"]] + assert events == [["got", 1], ["page", "TypeError"], ["page", "TypeError"]] # A Worker with no Workflows and a Store with a bad option. @@ -1781,7 +1913,7 @@ class ReadAppendCloseWorkflow: async def run(self, source_id: str, out_id: str) -> list[str]: source = workflow.get_stream(StreamRef(id=source_id), body_type=str) out = workflow.get_stream(StreamRef(id=out_id), body_type=str) - got = [record.body async for record in source.iter()] + got = [record.body async for record in source.iter_read_items()] await out.append(*got) await out.close() return got @@ -1828,7 +1960,7 @@ def values(name: str) -> list[Any]: assert update.attributes["task_queue"] == worker.task_queue -# The example of `iter`: a Workflow starts each tool as its call arrives. +# The example of the README: a Workflow starts each tool as its call arrives. @dataclass @@ -1878,7 +2010,7 @@ async def run(self, events_id: str, status_id: str) -> list[str]: events = workflow.get_stream(StreamRef(id=events_id), body_type=ModelEvent) status = workflow.get_stream(StreamRef(id=status_id), body_type=str) tools = [] - async for record in events.iter(): + async for record in events.iter_read_items(wait=timedelta(seconds=10)): event = record.body if event.kind == "tool_call": assert event.tool_call is not None @@ -1948,9 +2080,9 @@ async def test_read_with_a_read_ahead_of_one_byte_gives_every_record(client: Cli stream_client = Client(**config) bodies = [f"record-{index}" for index in range(20)] ref = await _filled_stream(stream_client, bodies) - _, result = await _run(stream_client, ReadOnceWorkflow, ReadArgs(ref.id)) + _, result = await _run(stream_client, ReadPagesWorkflow, ReadArgs(ref.id)) assert result.bodies == bodies - assert (result.stop, result.done) == ("Done", True) + assert result.dones[-1] is True # The batch window. @@ -1992,7 +2124,10 @@ async def run(self, stream_id: str) -> list[str]: append_five_apart, stream_id, start_to_close_timeout=timedelta(seconds=30) ) stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) - got = [record.body async for record in stream.iter()] + got = [ + record.body + async for record in stream.iter_read_items(wait=timedelta(seconds=10)) + ] await writer return got @@ -2037,7 +2172,7 @@ def __init__(self) -> None: async def read_then_wait(self, stream_id: str) -> list[str]: stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) - got = [record.body async for record in stream.iter(idle_timeout=timedelta())] + got = [record.body async for record in stream.iter_read_items()] await workflow.wait_condition(lambda: self.go) return got @@ -2118,3 +2253,51 @@ async def test_lost_records_fail_the_workflow_with_the_worker_option(client: Cli async def test_lost_records_fail_the_workflow_with_the_defn_option(client: Client): await _replay_with_records_gone(client, ReadThenWaitFailsWorkflow) + + +# A Workflow outbound interceptor sees each page that Workflow code reads. + +_intercepted: list[tuple[str, Any]] = [] + + +class StreamCallInterceptor(Interceptor): + def workflow_interceptor_class( + self, input: WorkflowInterceptorClassInput + ) -> type[WorkflowInboundInterceptor]: + return _StreamCallInbound + + +class _StreamCallInbound(WorkflowInboundInterceptor): + def init(self, outbound: WorkflowOutboundInterceptor) -> None: + super().init(_StreamCallOutbound(outbound)) + + +class _StreamCallOutbound(WorkflowOutboundInterceptor): + async def start_system_nexus_operation( + self, input: StartNexusOperationInput[Any, Any] + ) -> workflow.NexusOperationHandle[Any]: + _intercepted.append((input.operation_name, input.input)) + return await super().start_system_nexus_operation(input) + + +async def test_workflow_interceptor_sees_each_page_read(client: Client): + _intercepted.clear() + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c", "d", "e"]) + async with new_worker( + stream_client, ReadPagesWorkflow, interceptors=[StreamCallInterceptor()] + ) as worker: + result = await stream_client.execute_workflow( + ReadPagesWorkflow.run, + ReadArgs(ref.id, max_records=2), + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert result.page_sizes == [2, 2, 1] + assert [name for name, _ in _intercepted] == ["ReadRecords"] * 3 + # Each page starts after the last Record of the page before it. + assert [request.from_offset for _, request in _intercepted] == [ + None, + bytes.fromhex(result.offsets[1]), + bytes.fromhex(result.offsets[3]), + ] diff --git a/tests/nexus/test_stream_store_workflow.py b/tests/nexus/test_stream_store_workflow.py index 053ecaddc..362d7a426 100644 --- a/tests/nexus/test_stream_store_workflow.py +++ b/tests/nexus/test_stream_store_workflow.py @@ -143,11 +143,10 @@ async def run(self, stream_id: str) -> None: @workflow.defn -class RefusedOperationWorkflow: +class NexusClientReadWorkflow: @workflow.run async def run(self, operation: str) -> str: - # The Workflow `Stream` has no `read`, so the test starts its operation through - # a Nexus client, as Workflow code still can. + # Workflow code can also start the operation of `read` through a Nexus client. requests: dict[str, Any] = { "ReadRecords": ReadRecordsRequest(ref=StreamRef[str](id="stream")), } @@ -155,11 +154,14 @@ async def run(self, operation: str) -> str: service=StreamService, endpoint=TEMPORAL_SYSTEM_ENDPOINT ) try: - await nexus_client.start_operation(operation, requests[operation]) + handle = await nexus_client.start_operation(operation, requests[operation]) + await handle + except StreamError as err: + return StreamErrorReason.Name(err.reason) except ApplicationError as err: assert err.non_retryable return err.message - return "started" + return "read" @workflow.defn @@ -665,25 +667,41 @@ async def test_workflow_create_without_stream_config_fails_the_workflow( ) -async def test_workflow_read_fails_with_store(client: Client): - stream_client = _with_store(client, MemoryStreamStore()) - async with new_worker(stream_client, RefusedOperationWorkflow) as worker: - result = await stream_client.execute_workflow( - RefusedOperationWorkflow.run, - "ReadRecords", - id=f"workflow-{uuid.uuid4()}", - task_queue=worker.task_queue, - ) - assert result == ( - "Workflow code reads a Stream with `iter` of the Workflow Stream, not " - "with the ReadRecords operation." +async def test_workflow_read_through_a_nexus_client_needs_a_runtime( + client: Client, +): + async def read_with(store: StreamStore) -> str: + stream_client = _with_store(client, store) + async with new_worker(stream_client, NexusClientReadWorkflow) as worker: + return await stream_client.execute_workflow( + NexusClientReadWorkflow.run, + "ReadRecords", + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + + # Core reads a Store with a Rust runtime, as for `read`. Nobody created the Stream. + assert await read_with(MemoryStreamStore()) == "STREAM_ERROR_REASON_NOT_FOUND" + assert await read_with(DictStreamStore()) == ( + "The Stream Store 'dict' has no Direct read, since it is written in Python. " + "Read the Stream in an Activity, or use a Store that runs on the bridge, such " + "as the Redis Store or MemoryStreamStore." ) -def test_workflow_stream_has_append_and_close_and_no_read(): - for name in ("append", "close", "describe", "__aenter__", "__aexit__"): +def test_workflow_stream_has_the_calls_of_a_client_stream(): + for name in ( + "append", + "close", + "describe", + "read", + "iter_read", + "iter_read_items", + "__aenter__", + "__aexit__", + ): assert hasattr(WorkflowStream, name) - assert not hasattr(WorkflowStream, "read") + assert not hasattr(WorkflowStream, "iter") async def test_workflow_stream_store_error_reaches_workflow_code(client: Client): From eba8804361e8b56b2fbddf401344417ec037f785 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 23:40:53 -0700 Subject: [PATCH 50/59] Documented the Workflow read with `iter_read_items`. The stops, `last_offset` of a read and the idle default of one minute are gone. A loop on a quiet Stream ends with `iter_read` after an empty page, and the costs that still hold stay. --- changelog/added/patient-river-heron.md | 2 +- temporalio/client/_stream_store.py | 46 +++++++++++++++++-- temporalio/contrib/streams/README.md | 63 +++++++++++--------------- temporalio/worker/_replayer.py | 14 +++--- 4 files changed, 77 insertions(+), 48 deletions(-) diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md index 0841527d4..b46b3f147 100644 --- a/changelog/added/patient-river-heron.md +++ b/changelog/added/patient-river-heron.md @@ -1,4 +1,4 @@ -Add `iter` to the Workflow `Stream`, so Workflow code reads each Record of a Stream as it arrives. By default a read ends with `Idle` after 1 minute with no new Record, so code that reads a Stream that can be quiet for longer checks `done` after the loop or sets a longer `idle_timeout`. A read that waits on a quiet Stream holds its Workflow Task open, with one Workflow Task about every 8 s. Core reads the Store inside the Workflow Task, and a read costs one marker in History for each Workflow Task that it spans. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). With `HISTORY`, one Workflow Task's marker holds at most 1 MiB of Records, and the read goes on in the next task. +Add `read`, `iter_read` and `iter_read_items` to the Workflow `Stream`, so Workflow code reads a Stream with the same calls and pages as a client. Core reads the Store inside the Workflow Task. A loop that asks for its next page in the same activation keeps one read open, and a read that no code waits on closes before the activation completes, so it never holds a Workflow Task open. With a `wait`, a quiet Stream gives empty pages, so a loop on a Stream that can be quiet for long uses `iter_read` and leaves after an empty page. A call that waits on a quiet Stream holds its Workflow Task open, with one Workflow Task about every 8 s. The calls of one Workflow Task write one marker in History. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). With `HISTORY`, one Workflow Task's marker holds at most 1 MiB of Records, and the read goes on in the next task. A Workflow's append on the Redis Store or `MemoryStreamStore` now runs in Core inside the Workflow Task, with no Local Activity. It writes as the Workflow, with its Workflow Task as the epoch. A Workflow's close on the Redis Store or `MemoryStreamStore` now runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close: a graceful shutdown of the Worker waits at most 10 s for a close that the code doesn't await, and a Worker that dies when the Workflow ends loses it. An append and a close of Workflow code always carry the Workflow Task as their epoch. An append is at-least-once: after a Worker crash or an eviction, the rerun of its task can write a second copy with the epoch of the new task. Core records metrics for the Direct calls of a Stream, such as `temporal_stream_read_total`, `temporal_stream_read_records` and `temporal_stream_close_latency`, each with the attribute `stream_store`, and `temporal_stream_close_abandoned` for a close that a graceful shutdown gave up. diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index e6a97b396..0859bed32 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -109,7 +109,7 @@ class StreamStore(ABC): subclassed in Python. A Store that is written in Python works for a client and an Activity, for example as a double in a test. Workflow code loses the Direct calls on it. An append and a close of Workflow code then run in a Local Activity, with one - marker in History for each call, and ``iter`` fails the Workflow, so Workflow code + marker in History for each call, and a read fails the Workflow, so Workflow code can't read its Streams. Such a Store implements :py:attr:`name` and the five operations. Each operation @@ -496,6 +496,44 @@ class StreamConfig: fails after a close, the Stream stays closed. The retry of the task then can't append to it. + Workflow code reads with the same ``read``, ``iter_read`` and ``iter_read_items`` as + a client, and gets the same pages. On a Store with a Rust runtime, Core reads the + Store inside the Workflow Task. A loop that asks for its next page in the same + activation keeps one read open in Core. A read that no code waits on closes before + the activation completes, so it never holds a Workflow Task open. + + .. code-block:: python + + async for record in events.iter_read_items(wait=timedelta(seconds=10)): + if record.body.kind == "tool_call": + tools.append( + workflow.start_activity( + run_tool, + record.body.tool_call, + start_to_close_timeout=timedelta(minutes=1), + ) + ) + + The loop ends after the page that is ``done``, when the Stream is closed and the + read gave its last Record. With a ``wait``, a quiet Stream gives empty pages, and + the loop goes on. So a loop on a Stream that can be quiet for long uses + ``iter_read`` and leaves after an empty page. A read has these costs. + + * A call keeps its Workflow Task open while it waits for its page. A Signal, an + Update, a cancel or an Activity result reaches the Workflow at the next Workflow + Task, at the heartbeat of the task at the latest. That is about 8 s with the + default task timeout of 10 s. + * The calls of one Workflow Task write one marker in History, whatever the number of + Records. + * With the default :py:attr:`StreamReplaySource.STORE`, History holds no Record. A + replay reads the Records from the Store again, so they must stay in the Store for + as long as the Workflow can replay. + * A replay on a Worker with no cache reads the Store again on every Workflow Task. + + On a Store written in Python, a read fails the Workflow with a non-retryable + :py:class:`temporalio.exceptions.ApplicationError`. Code in a Query handler or an + Update validator can't read. + .. warning:: This API is experimental and subject to change. """ @@ -552,8 +590,8 @@ class StreamConfig: workflow_read_ahead_bytes: int | None = None """The most bytes of one read's Records that wait in the Worker for Workflow code. - A read of Workflow code with ``iter`` stops asking the Store while this many bytes of - its Records wait in the Worker's memory for Workflow code to take them. The read goes + A read of Workflow code stops asking the Store while this many bytes of its Records + wait in the Worker's memory for Workflow code to take them. The read goes on when Workflow code takes Records. The first page of a read always comes. ``None`` takes Core's default of 4 MiB. Set a larger value for large Records, and a smaller one for a Worker with little memory, since each open read can hold this much. @@ -562,7 +600,7 @@ class StreamConfig: workflow_read_batch_window: timedelta | None = None """The least time between two batches that one read gives to Workflow code. - A read of Workflow code with ``iter`` gives its first batch immediately. After that it + A read of Workflow code gives its first batch immediately. After that it gives Workflow code a batch at most once in this window, with every Record that came in the window. So a busy Stream doesn't activate the Workflow for each Record. A Record waits at most one window. ``None`` takes Core's default of 50 ms, and zero is no diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 39da835d9..78f632e73 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -47,9 +47,8 @@ replacement of the first attempt's Records. The helpers run on the Streams API, so they follow how Workflow code reads, appends and closes. On a Store with a Rust runtime, such as the Redis Store or `MemoryStreamStore`, Core -reads, appends and closes a Stream itself, with no Activity. A read is `iter` of the Workflow -`Stream`. It gives each Record as it arrives and ends at a stop. By default a read ends with -`Idle` after 1 minute with no new Record, and has no other bound. +reads, appends and closes a Stream itself, with no Activity. Workflow code reads with the same +`read`, `iter_read` and `iter_read_items` as a client, and gets the same pages. Here the Workflow reads the events of a model's answer while the model Activity still runs, and starts each tool as its call arrives. @@ -58,7 +57,7 @@ and starts each tool as its call arrives. events = workflow.get_stream(events_ref, body_type=ModelEvent) status = workflow.get_stream(status_ref, body_type=str) tools = [] -async for record in events.iter(): +async for record in events.iter_read_items(wait=timedelta(seconds=10)): event = record.body if event.kind == "tool_call": # The tool starts now, before the model ends its answer. @@ -71,49 +70,41 @@ async for record in events.iter(): results = await asyncio.gather(*tools) ``` -When the Stream can be quiet for more than a minute, as a model that thinks for long can be, -check `done` after the loop, or set a longer `idle_timeout`. - -`iter` takes `from_offset` and four bounds, `timeout`, `idle_timeout`, `max_records` and -`max_bytes`. `idle_timeout` is 1 minute by default, and the others have no bound. An -`idle_timeout` ends a read with `Idle` when no Record comes for that long, and zero gives -what is in the Stream and doesn't wait. After a bound ended a read, a new read from -`last_offset` goes on where it stopped. This loop waits with a timer while the Stream is quiet. +The loop ends after the page that is `done`, when the Stream is closed and the read gave its +last Record. With no `wait`, a call gives what is in the Stream, and the loop ends at the first +empty page. With a `wait`, a call on a quiet Stream gives an empty page after the wait, and +the loop asks again. So a loop on a Stream that can be quiet for long uses `iter_read` and +leaves after an empty page. A later loop from the `last_offset` of that page goes on there. ```python last = None -while True: - reading = tokens.iter(from_offset=last, idle_timeout=timedelta(seconds=5)) - async for record in reading: +async for page in tokens.iter_read(from_offset=last, wait=timedelta(seconds=30)): + for record in page.records: await shown.append(record.body.text) - last = reading.last_offset - if reading.done: + last = page.last_offset + if not page.records: + # The Stream was quiet for 30 s, or it is done. break - await workflow.sleep(timedelta(minutes=1)) ``` -After the loop, `stop` tells which stop ended the read, and `done` is true when -the Stream is closed and the read gave its last Record. That stop is `Done` also when the -last Record filled a cap. The idle time counts only while the read waits for the Store. A -new read from `last_offset` loses and repeats no Record, also after `aclose` or a `break`. A -read that the code leaves with `break` and doesn't close runs on in Core until one of its -stops. - -A read costs one marker in History for each Workflow Task that it spans, whatever the number -of Records. It has three other costs. - -- A read keeps its Workflow Task open. A Signal, an Update, a cancel or an Activity result - reaches the Workflow at the next Workflow Task, at the heartbeat of the task at the latest. - That is about 8 s with the default task timeout of 10 s. A read that waits on a quiet - Stream holds its task open the whole time, with one Workflow Task about every 8 s, until - its `idle_timeout` ends it with `Idle`. +Core keeps one read open while a loop asks for its next page in the same activation. A loop +that awaits something else between two pages, as the `append` above does, lets the read close +before the activation completes, and its next page starts a new read. A read that no code +waits on never holds a Workflow Task open. + +The calls of one Workflow Task write one marker in History, whatever the number of Records. A +read has three other costs. + +- A call keeps its Workflow Task open while it waits for its page. A Signal, an Update, a + cancel or an Activity result reaches the Workflow at the next Workflow Task, at the + heartbeat of the task at the latest. That is about 8 s with the default task timeout of + 10 s. A call with a `wait` on a quiet Stream holds its task open for the whole wait. - By default, `StreamReplaySource.STORE`, the marker holds a range of offsets, a count and a digest, and no Record. A replay reads the Store again, so the Records must stay in the Store for as long as the Workflow can replay. The option `workflow_read_replay_from=StreamReplaySource.HISTORY` of `StreamConfig` puts the Records - into the marker, so a replay needs no Store. Then a read with no `max_bytes` gets a cap of - 256 KiB, and one Workflow Task's marker holds at most 1 MiB of Records. Past that, the task - completes and the read goes on in the next one. + into the marker, so a replay needs no Store. Then one Workflow Task's marker holds at most + 1 MiB of Records. Past that, the task completes and the read goes on in the next one. - A replay on a Worker with no cache reads the Store again on every Workflow Task. While Workflow code is busy, a read holds at most 4 MiB of its Records in the Worker, then diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index cf886c8c2..9e92b0314 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -77,13 +77,13 @@ def __init__( needs it. With no Stream config, Workflow code gets an error in place of that call, and the replay fails with a nondeterminism error. A replay calls no method of its Stores from Python. An append or a close that Core made needs no Stream - config, and a replay closes nothing. A read of Workflow code with ``iter`` that - ran under the option ``HISTORY`` of ``workflow_read_replay_from`` needs none - either. That holds also when the ref names no Store, since Core takes the Store - from the history. A read that ran under the default, - ``StreamReplaySource.STORE``, has no Records in the history. Core reads them from the Store again, so the replay - needs this config and Stores that still have the Records. Without them the - replay fails with a :py:class:`RuntimeError` whose message has + config, and a replay closes nothing. A read of Workflow code that ran under the + option ``HISTORY`` of ``workflow_read_replay_from`` needs none either. That holds + also when the ref has no Store, since Core takes the Store from the history. A + read that ran under the default, ``StreamReplaySource.STORE``, has no Records in + the history. Core reads them from the Store again, so the replay needs this + config and Stores that still have the Records. Without them the replay fails + with a :py:class:`RuntimeError` whose message has ``StreamReplayError``, the Stream and the cause. Note, unlike the worker, for the replayer the workflow_task_executor From 5fe21e2a460e3440ad1a55c0d60312d124c83207 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Mon, 16 Nov 2026 23:42:33 -0700 Subject: [PATCH 51/59] Fixed the type checks of the Workflow read and its tests. mypy wants a type for the handle of a Nexus read and a mapping for the read cases, and basedpyright can't see that the instance uses `_DirectRead`. --- temporalio/worker/_stream_store.py | 2 +- tests/nexus/test_stream_direct.py | 30 ++++++++++------------- tests/nexus/test_stream_store_workflow.py | 4 ++- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index 0d5152bfe..6c2d182b1 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -213,7 +213,7 @@ class _ReadKey: wait: timedelta | None -class _DirectRead: +class _DirectRead: # pyright: ignore[reportUnusedClass] """One Core read of a Stream, which serves the ``read`` calls of Workflow code. Core gives its Records in batches. Each ``read`` call takes one page of them, and the diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 1792b4c4e..b09af2267 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -586,21 +586,24 @@ async def test_loop_that_awaits_an_append_closes_and_starts_the_read( # What `read` checks, and where it can't run. +# A client's read clamps each of these, or refuses it, and a Workflow read does the same. +_READ_LIMIT_CASES: list[dict[str, Any]] = [ + {"max_bytes": -1}, + {"max_records": 0}, + {"max_records": -1}, + {"max_records": 5000}, + {"wait": timedelta(seconds=-1)}, + {"wait": timedelta(seconds=120)}, +] + + @workflow.defn class ReadLimitsWorkflow: @workflow.run async def run(self, ref: StreamRef[str]) -> list[Any]: stream = workflow.get_stream(ref, body_type=str) results: list[Any] = [] - cases: list[dict[str, Any]] = [ - {"max_bytes": -1}, - {"max_records": 0}, - {"max_records": -1}, - {"max_records": 5000}, - {"wait": timedelta(seconds=-1)}, - {"wait": timedelta(seconds=120)}, - ] - for arguments in cases: + for arguments in _READ_LIMIT_CASES: try: page = await stream.read(**arguments) results.append(len(page.records)) @@ -615,14 +618,7 @@ async def test_read_takes_the_limits_of_a_client_read(client: Client): _, results = await _run(stream_client, ReadLimitsWorkflow, ref) stream = stream_client.get_stream(ref, body_type=str) expected: list[Any] = [] - for arguments in [ - {"max_bytes": -1}, - {"max_records": 0}, - {"max_records": -1}, - {"max_records": 5000}, - {"wait": timedelta(seconds=-1)}, - {"wait": timedelta(seconds=120)}, - ]: + for arguments in _READ_LIMIT_CASES: try: page = await stream.read(**arguments) expected.append(len(page.records)) diff --git a/tests/nexus/test_stream_store_workflow.py b/tests/nexus/test_stream_store_workflow.py index 362d7a426..9b4b95c3c 100644 --- a/tests/nexus/test_stream_store_workflow.py +++ b/tests/nexus/test_stream_store_workflow.py @@ -154,7 +154,9 @@ async def run(self, operation: str) -> str: service=StreamService, endpoint=TEMPORAL_SYSTEM_ENDPOINT ) try: - handle = await nexus_client.start_operation(operation, requests[operation]) + handle: workflow.NexusOperationHandle[ + Any + ] = await nexus_client.start_operation(operation, requests[operation]) await handle except StreamError as err: return StreamErrorReason.Name(err.reason) From 5bdbd90e220fc70480b3a592a7f465b1613ef3ea Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Tue, 17 Nov 2026 00:10:26 -0700 Subject: [PATCH 52/59] Tested a decision that depends on the activations of a read. A Workflow counts how often it wakes while it reads, and starts an Activity at a count. A replay starts it at the same Record, since the marker has the activation of each page. With other activations in the marker, the replay is a non-determinism error. --- tests/nexus/test_stream_direct.py | 143 ++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index b09af2267..e221ebc32 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -587,6 +587,149 @@ async def test_loop_that_awaits_an_append_closes_and_starts_the_read( # A client's read clamps each of these, or refuses it, and a Workflow read does the same. +# The example of the design review. A second task counts how often the Workflow wakes, and +# the read loop starts an Activity when that count went up by two. So what the Workflow does +# depends on the activations that gave it its Records, not only on the Records. + + +@dataclass +class WakeCountResult: + started_at: int + wakes: list[int] + + +@workflow.defn +class WakeCountWorkflow: + def __init__(self) -> None: + self._wakes = 0 + + def _count_a_wake(self) -> bool: + # The Workflow's event loop checks each condition when it has run every ready task. + # This one never holds, so it counts those checks. + self._wakes += 1 + return False + + @workflow.run + async def run(self, stream_id: str) -> WakeCountResult: + writer = workflow.start_activity( + append_five_slowly, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + counter = asyncio.create_task(workflow.wait_condition(self._count_a_wake)) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + wakes: list[int] = [] + started_at = -1 + async for record in stream.iter_read_items(wait=timedelta(seconds=10)): + wakes.append(self._wakes) + if started_at < 0 and self._wakes >= wakes[0] + 2: + started_at = len(wakes) - 1 + await workflow.execute_activity( + tag, record.body, start_to_close_timeout=timedelta(seconds=10) + ) + counter.cancel() + await writer + return WakeCountResult(started_at, wakes) + + +def _pages_until_activity(history: WorkflowHistory, name: str) -> tuple[int, int]: + """Give the Records and the activations of the pages up to an Activity's start. + + It counts the `core_stream` markers up to the Workflow Task that started the first + Activity called ``name``, with the marker of that task. + """ + records = activations = 0 + found = False + for event in history.events: + if found and event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED: + break + if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED: + scheduled = event.activity_task_scheduled_event_attributes + found = found or scheduled.activity_type.name == name + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + for entry in StreamMarkerData.FromString(payload.data).reads: + records += entry.count + if entry.page_activations: + # The first number is an index, and each later one is the gap to the page + # before it. + activations += 1 + sum(1 for gap in entry.page_activations[1:] if gap) + assert found, f"History has no Activity called {name}" + return records, activations + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_decision_on_the_wake_count_replays_the_same( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run( + stream_client, + WakeCountWorkflow, + ref.id, + activities=[append_five_slowly, tag], + max_cached_workflows=max_cached_workflows, + ) + # The Records came in more than one activation, and the Activity started at a later + # Record than the first one. + assert len(result.wakes) == 5 + assert result.wakes == sorted(result.wakes) + assert len(set(result.wakes)) > 1 + assert 1 <= result.started_at < 5 + history = await handle.fetch_history() + # The marker of the Workflow Task that started the Activity has every Record that the + # loop had by then, and the activation that gave each page. + records, activations = _pages_until_activity(history, "tag") + assert records >= result.started_at + 1 + assert activations > 1 + # A replay gives each page in the activation that got it, so the count and the + # Activity are the same. With no cache, each Workflow Task of the run above was such a + # replay too. + config = stream_client.config()["stream_config"] + await Replayer(workflows=[WakeCountWorkflow], stream_config=config).replay_workflow( + history + ) + + +async def test_replay_with_other_activations_is_a_nondeterminism_error(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run( + stream_client, + WakeCountWorkflow, + ref.id, + activities=[append_five_slowly, tag], + ) + assert result.started_at >= 1 + history = await handle.fetch_history() + assert _pages_until_activity(history, "tag")[1] > 1 + # Put every page of a read into the activation of its first page. The Workflow then + # wakes less often before it has the Record at which it started the Activity. + for event in history.events: + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + data = StreamMarkerData.FromString(payload.data) + for entry in data.reads: + gaps = [0] * (len(entry.page_activations) - 1) + del entry.page_activations[1:] + entry.page_activations.extend(gaps) + payload.data = data.SerializeToString() + config = stream_client.config()["stream_config"] + replay = await Replayer( + workflows=[WakeCountWorkflow], stream_config=config + ).replay_workflow(history, raise_on_replay_failure=False) + assert isinstance(replay.replay_failure, workflow.NondeterminismError) + + _READ_LIMIT_CASES: list[dict[str, Any]] = [ {"max_bytes": -1}, {"max_records": 0}, From b693a5834b6bcfbb2733563971e2fd0d64c2a975 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Tue, 17 Nov 2026 00:50:30 -0700 Subject: [PATCH 53/59] Wrote the review's example with a counter task for each Record. The earlier test counted the wakes with one condition. This one has the shape of the example, where each Record starts a task that counts and the loop starts an Activity when the count is 2. --- tests/nexus/test_stream_direct.py | 240 ++++++++++++++++++------------ 1 file changed, 142 insertions(+), 98 deletions(-) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index e221ebc32..9e2498939 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -583,132 +583,172 @@ async def test_loop_that_awaits_an_append_closes_and_starts_the_read( assert len(_read_seqs(await handle.fetch_history())) > 1 -# What `read` checks, and where it can't run. - - -# A client's read clamps each of these, or refuses it, and a Workflow read does the same. -# The example of the design review. A second task counts how often the Workflow wakes, and -# the read loop starts an Activity when that count went up by two. So what the Workflow does -# depends on the activations that gave it its Records, not only on the Records. +# The example of the design review: +# +# let mut i = 0; +# while let item = stream.next() { +# workflow.spawn(loop { yield_once.await; i++; }) +# if i == 2 { await execute_activity(...); } +# } +# +# Each Record starts one more task that counts the wakes of the Workflow. So whether the +# Activity starts, and at which Record, depends on the activations that gave the Workflow +# its Records, and not on the Records. + + +class _Wakes: + """Lets a task of a Workflow wait for the next wake of the Workflow. + + Python has no call for it. ``asyncio.sleep(0)`` resumes in the same activation, so a + loop of it never lets the activation end. + """ + def __init__(self) -> None: + self._new: list[asyncio.Future[None]] = [] + self._parked: list[asyncio.Future[None]] = [] + self._resumed = False + self._watch = asyncio.create_task(workflow.wait_condition(self._on_idle)) + + def _on_idle(self) -> bool: + # The event loop checks a condition each time it has no ready task. That is once at + # the end of a wake, and once more when the tasks that this resumed have run. The + # second check is in the same wake, so it resumes nothing. + if self._resumed: + self._resumed = False + self._parked += self._new + else: + self._resumed = bool(self._parked) + for future in self._parked: + if not future.done(): + future.set_result(None) + self._parked = self._new + self._new = [] + return False -@dataclass -class WakeCountResult: - started_at: int - wakes: list[int] + async def yield_once(self) -> None: + future: asyncio.Future[None] = asyncio.get_running_loop().create_future() + self._new.append(future) + await future @workflow.defn -class WakeCountWorkflow: +class CounterTasksWorkflow: def __init__(self) -> None: - self._wakes = 0 + self.i = 0 - def _count_a_wake(self) -> bool: - # The Workflow's event loop checks each condition when it has run every ready task. - # This one never holds, so it counts those checks. - self._wakes += 1 - return False + async def count(self, wakes: _Wakes) -> None: + while True: + await wakes.yield_once() + self.i += 1 @workflow.run - async def run(self, stream_id: str) -> WakeCountResult: - writer = workflow.start_activity( - append_five_slowly, stream_id, start_to_close_timeout=timedelta(seconds=30) - ) - counter = asyncio.create_task(workflow.wait_condition(self._count_a_wake)) + async def run(self, stream_id: str) -> list[int]: + wakes = _Wakes() + seen: list[int] = [] stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) - wakes: list[int] = [] - started_at = -1 - async for record in stream.iter_read_items(wait=timedelta(seconds=10)): - wakes.append(self._wakes) - if started_at < 0 and self._wakes >= wakes[0] + 2: - started_at = len(wakes) - 1 + async for item in stream.iter_read_items(wait=timedelta(seconds=30)): + asyncio.create_task(self.count(wakes)) + seen.append(self.i) + if self.i == 2: await workflow.execute_activity( - tag, record.body, start_to_close_timeout=timedelta(seconds=10) + tag, item.body, start_to_close_timeout=timedelta(seconds=10) ) - counter.cancel() - await writer - return WakeCountResult(started_at, wakes) + return seen + + +async def _read_calls(count: int) -> None: + """Wait until Workflow code has made this many `read` calls.""" + async def check() -> None: + assert [name for name, _ in _intercepted].count("ReadRecords") >= count -def _pages_until_activity(history: WorkflowHistory, name: str) -> tuple[int, int]: - """Give the Records and the activations of the pages up to an Activity's start. + await assert_eventually(check, interval=timedelta(milliseconds=20)) - It counts the `core_stream` markers up to the Workflow Task that started the first - Activity called ``name``, with the marker of that task. + +async def _run_counter_tasks( + client: Client, pages: list[list[str]], *, max_cached_workflows: int = 1000 +) -> tuple[list[int], WorkflowHistory]: + """Run the example, and give it each page of Records in an activation of its own. + + The last page must make the Workflow start its Activity. """ - records = activations = 0 - found = False - for event in history.events: - if found and event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED: - break - if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED: - scheduled = event.activity_task_scheduled_event_attributes - found = found or scheduled.activity_type.name == name - attributes = event.marker_recorded_event_attributes - if ( - event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED - or attributes.marker_name != "core_stream" - ): - continue - [payload] = attributes.details["data"].payloads - for entry in StreamMarkerData.FromString(payload.data).reads: - records += entry.count - if entry.page_activations: - # The first number is an index, and each later one is the gap to the page - # before it. - activations += 1 + sum(1 for gap in entry.page_activations[1:] if gap) - assert found, f"History has no Activity called {name}" - return records, activations + _intercepted.clear() + stream = await client.create_stream(id=f"stream-{uuid.uuid4()}", body_type=str) + # The Records that the Stream has when the read starts come in one page. + await stream.append(*pages[0]) + async with new_worker( + client, + CounterTasksWorkflow, + activities=[tag], + interceptors=[StreamCallInterceptor()], + max_cached_workflows=max_cached_workflows, + ) as worker: + handle = await client.start_workflow( + CounterTasksWorkflow.run, + stream.ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + for calls, page in enumerate(pages[1:], start=2): + # The loop asks for its next page in the activation that gave it the page + # before. A Record that comes after that call is in a later activation. + await _read_calls(calls) + await stream.append(*page) + + async def activity_started() -> None: + assert _scheduled_activities(await handle.fetch_history()) == ["tag"] + + await assert_eventually(activity_started) + await stream.close() + seen = await handle.result() + return seen, await handle.fetch_history() + + +_COUNTER_PAGES = [["record-0", "record-1"], ["record-2"], ["record-3"]] @pytest.mark.parametrize("max_cached_workflows", [1000, 0]) -async def test_decision_on_the_wake_count_replays_the_same( +async def test_counter_tasks_start_the_activity_by_the_activations( client: Client, max_cached_workflows: int ): stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, [], close=False) - handle, result = await _run( - stream_client, - WakeCountWorkflow, - ref.id, - activities=[append_five_slowly, tag], - max_cached_workflows=max_cached_workflows, + seen, history = await _run_counter_tasks( + stream_client, _COUNTER_PAGES, max_cached_workflows=max_cached_workflows ) - # The Records came in more than one activation, and the Activity started at a later - # Record than the first one. - assert len(result.wakes) == 5 - assert result.wakes == sorted(result.wakes) - assert len(set(result.wakes)) > 1 - assert 1 <= result.started_at < 5 - history = await handle.fetch_history() - # The marker of the Workflow Task that started the Activity has every Record that the - # loop had by then, and the activation that gave each page. - records, activations = _pages_until_activity(history, "tag") - assert records >= result.started_at + 1 - assert activations > 1 - # A replay gives each page in the activation that got it, so the count and the - # Activity are the same. With no cache, each Workflow Task of the run above was such a - # replay too. + # The first two Records start a task each, and both tasks count the next wake. The loop + # reads `i` before the tasks of a wake run, so `i` is 2 at the fourth Record. + assert seen == [0, 0, 0, 2] + # The marker of the Workflow Task that started the Activity has the three pages, and + # each page is in an activation of its own. + [entry] = _read_entries(history)[0] + assert list(entry.page_counts) == [2, 1, 1] + assert all(gap > 0 for gap in entry.page_activations[1:]) + # A replay gives each page in the activation that got it, so `i` is the same at each + # Record. With no cache, the Workflow Tasks after the first one were such replays too. config = stream_client.config()["stream_config"] - await Replayer(workflows=[WakeCountWorkflow], stream_config=config).replay_workflow( - history + await Replayer( + workflows=[CounterTasksWorkflow], stream_config=config + ).replay_workflow(history) + + +async def test_counter_tasks_with_one_page_start_no_activity(client: Client): + stream_client = _stream_client(client) + bodies = [body for page in _COUNTER_PAGES for body in page] + ref = await _filled_stream(stream_client, bodies) + handle, seen = await _run( + stream_client, CounterTasksWorkflow, ref.id, activities=[tag] ) + # With the same Records in one activation, no task counts a wake between two of them. + assert seen == [0, 0, 0, 0] + assert _scheduled_activities(await handle.fetch_history()) == [] async def test_replay_with_other_activations_is_a_nondeterminism_error(client: Client): stream_client = _stream_client(client) - ref = await _filled_stream(stream_client, [], close=False) - handle, result = await _run( - stream_client, - WakeCountWorkflow, - ref.id, - activities=[append_five_slowly, tag], - ) - assert result.started_at >= 1 - history = await handle.fetch_history() - assert _pages_until_activity(history, "tag")[1] > 1 - # Put every page of a read into the activation of its first page. The Workflow then - # wakes less often before it has the Record at which it started the Activity. + seen, history = await _run_counter_tasks(stream_client, _COUNTER_PAGES) + assert seen == [0, 0, 0, 2] + # Put every page of a read into the activation of its first page. `i` then stays 0, and + # the replay does not start the Activity that History has. for event in history.events: attributes = event.marker_recorded_event_attributes if ( @@ -725,11 +765,15 @@ async def test_replay_with_other_activations_is_a_nondeterminism_error(client: C payload.data = data.SerializeToString() config = stream_client.config()["stream_config"] replay = await Replayer( - workflows=[WakeCountWorkflow], stream_config=config + workflows=[CounterTasksWorkflow], stream_config=config ).replay_workflow(history, raise_on_replay_failure=False) assert isinstance(replay.replay_failure, workflow.NondeterminismError) +# What `read` checks, and where it can't run. + + +# A client's read clamps each of these, or refuses it, and a Workflow read does the same. _READ_LIMIT_CASES: list[dict[str, Any]] = [ {"max_bytes": -1}, {"max_records": 0}, From 7e9dec37f2f38d84aac9c185416ae62350acf701 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Tue, 17 Nov 2026 01:38:50 -0700 Subject: [PATCH 54/59] Added tests for taking one Stream page in several activations. An Activity between Records of one page is a later Workflow Task. The marker stays on the task that received the page. --- tests/nexus/test_stream_direct.py | 104 ++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 9e2498939..03d2c0e77 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -770,6 +770,110 @@ async def test_replay_with_other_activations_is_a_nondeterminism_error(client: C assert isinstance(replay.replay_failure, workflow.NondeterminismError) +# The Records of one page, taken in several activations. Core records the page +# when it gives it. An Activity between two Records is a later Workflow Task, +# and that task has no marker. + + +@workflow.defn +class ActivityForEachRecordWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [] + async for item in stream.iter_read_items(): + got.append( + await workflow.execute_activity( + tag, item.body, start_to_close_timeout=timedelta(seconds=10) + ) + ) + return got + + +@workflow.defn +class OneRecordPerReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [] + offset = None + while True: + page = await stream.read(from_offset=offset, max_records=1) + for record in page.records: + got.append( + await workflow.execute_activity( + tag, + record.body, + start_to_close_timeout=timedelta(seconds=10), + ) + ) + offset = page.last_offset + if page.done or not page.records: + return got + + +async def _one_page_across_activations( + client: Client, workflow_type: type[Any], *, max_cached_workflows: int +) -> None: + ref = await _filled_stream(client, ["a", "b", "c", "d"]) + handle, got = await _run( + client, + workflow_type, + ref.id, + activities=[tag], + max_cached_workflows=max_cached_workflows, + ) + assert got == [f"tagged {body}" for body in "abcd"] + history = await handle.fetch_history() + # The last Activity's result is a task of its own, and that task completes + # the Workflow. It records nothing, so the marker stays in the first task. + assert _completed_tasks(history) == 5 + assert _scheduled_activities(history) == ["tag", "tag", "tag", "tag"] + task = 0 + marker_task = None + for event in history.events: + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED: + task += 1 + elif ( + event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + and event.marker_recorded_event_attributes.marker_name == "core_stream" + ): + assert marker_task is None + marker_task = task + assert marker_task == 1 + entries = _read_entries(history) + assert len(entries) == 1 + [entry] = entries[0] + assert list(entry.page_counts) == [4] + assert list(entry.page_activations) == [1] + config = client.config()["stream_config"] + await Replayer(workflows=[workflow_type], stream_config=config).replay_workflow( + history + ) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_one_page_across_activities_keeps_its_marker( + client: Client, max_cached_workflows: int +): + await _one_page_across_activations( + _stream_client(client), + ActivityForEachRecordWorkflow, + max_cached_workflows=max_cached_workflows, + ) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_one_record_reads_of_one_page_keep_its_marker( + client: Client, max_cached_workflows: int +): + await _one_page_across_activations( + _stream_client(client), + OneRecordPerReadWorkflow, + max_cached_workflows=max_cached_workflows, + ) + + # What `read` checks, and where it can't run. From 844a42d974c21740a5fe4c87017deb7480825aec Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Tue, 17 Nov 2026 02:44:12 -0700 Subject: [PATCH 55/59] Moved Core to the retry of a Store outage for Direct calls. Core tries a Direct read, append or close again while its Store is unavailable. A replay whose Store stays down fails only its Workflow Task. --- temporalio/bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 2e8f906cf..39315d79f 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 2e8f906cff399bcf3def95acbcec5e4366adf568 +Subproject commit 39315d79f3fcb540b0bfe6ae8ab6284c4f83fde6 From ac0e58003d59786c3ff27cb5423d4b006694556d Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Tue, 17 Nov 2026 02:53:10 -0700 Subject: [PATCH 56/59] Tested Direct calls and a replay through a short Store outage. A proxy in front of Redis drops each connection while it is down. The tests check that an append and a read give each Record once, and that a replay fails only its task. --- tests/nexus/test_stream_direct.py | 264 +++++++++++++++++++++++++++++- 1 file changed, 263 insertions(+), 1 deletion(-) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 03d2c0e77..c4c53e4e8 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -3,11 +3,13 @@ from __future__ import annotations import asyncio +import os +import urllib.parse import uuid from collections.abc import AsyncGenerator, Sequence from dataclasses import dataclass from datetime import timedelta -from typing import Any +from typing import Any, Self import pytest @@ -23,6 +25,7 @@ Client, StreamConfig, StreamReplaySource, + WorkflowExecutionStatus, WorkflowFailureError, WorkflowHandle, WorkflowHistory, @@ -2588,3 +2591,262 @@ async def test_workflow_interceptor_sees_each_page_read(client: Client): bytes.fromhex(result.offsets[1]), bytes.fromhex(result.offsets[3]), ] + + +# A Store outage that Core tries again, made with a proxy in front of Redis. + +_REDIS_URL = os.environ.get("STREAMS_REDIS_URL") + +_needs_redis = pytest.mark.skipif( + not _REDIS_URL, reason="STREAMS_REDIS_URL is not set to a Redis server" +) + + +async def _copy(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + except ConnectionError: + pass + + +class _StoreOutage: + """A TCP proxy in front of Redis that a test takes down and brings up. + + Down, it drops each open connection and each new one, so each call of the Store + fails with `StorageUnavailable`. Redis itself keeps running, so the tests that + share it in other processes see no outage. `refused` counts the connections that + it dropped while down. + """ + + def __init__(self, url: str) -> None: + self._parts = urllib.parse.urlsplit(url) + self._server: asyncio.Server | None = None + self._writers: set[asyncio.StreamWriter] = set() + self._down = False + self.url = "" + self.refused = 0 + + async def __aenter__(self) -> Self: + self._server = await asyncio.start_server(self._serve, "127.0.0.1", 0) + port = self._server.sockets[0].getsockname()[1] + credentials, at, _ = self._parts.netloc.rpartition("@") + netloc = f"{credentials}{at}127.0.0.1:{port}" + self.url = urllib.parse.urlunsplit(self._parts._replace(netloc=netloc)) + return self + + async def __aexit__(self, *_: object) -> None: + self.take_down() + if self._server: + self._server.close() + + def take_down(self) -> None: + self._down = True + for writer in list(self._writers): + writer.close() + + def bring_up(self) -> None: + self._down = False + + async def _serve( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + if self._down: + self.refused += 1 + writer.close() + return + try: + redis_reader, redis_writer = await asyncio.open_connection( + self._parts.hostname or "127.0.0.1", self._parts.port or 6379 + ) + except OSError: + writer.close() + return + self._writers.update((writer, redis_writer)) + copies = [ + asyncio.create_task(_copy(reader, redis_writer)), + asyncio.create_task(_copy(redis_reader, writer)), + ] + try: + await asyncio.wait(copies, return_when=asyncio.FIRST_COMPLETED) + finally: + for each in (writer, redis_writer): + self._writers.discard(each) + each.close() + await asyncio.gather(*copies) + + +def _redis_client(client: Client, url: str) -> Client: + from temporalio.contrib.redis.streams import RedisStreamStore + + # A short response timeout keeps a call that the outage cuts from waiting long. + store = RedisStreamStore( + url, + key_prefix=f"test-{uuid.uuid4()}", + response_timeout=timedelta(seconds=1), + ) + return _stream_client(client, store) + + +async def _store_answers(stream: Any) -> None: + # A client call has no retry, so the test waits until the Store answers again. + try: + await stream.describe() + except StreamError as err: + raise AssertionError(f"The Store doesn't answer: {err}") from err + + +@_needs_redis +async def test_append_during_a_short_outage_appends_once(client: Client): + assert _REDIS_URL + async with _StoreOutage(_REDIS_URL) as outage: + stream_client = _redis_client(client, outage.url) + ref = await _filled_stream(stream_client, [], close=False) + stream = stream_client.get_stream(ref, body_type=str) + async with new_worker( + stream_client, AppendWorkflow, max_cached_workflows=0 + ) as worker: + outage.take_down() + handle: WorkflowHandle[Any, Any] = await stream_client.start_workflow( + AppendWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + # The client gets the error that Core keeps from Workflow code. + with pytest.raises(StreamError) as err: + await stream.describe() + assert ( + err.value.reason + == StreamErrorReason.STREAM_ERROR_REASON_STORAGE_UNAVAILABLE + ) + + async def task_started() -> None: + events = (await handle.fetch_history()).events + assert any( + event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_STARTED + for event in events + ) + + await assert_eventually(task_started) + refused = outage.refused + await asyncio.sleep(1) + # The task waits for the append, which Core tries again. + assert outage.refused > refused + assert _completed_tasks(await handle.fetch_history()) == 0 + outage.bring_up() + offsets = await asyncio.wait_for(handle.result(), timeout=30) + records = await _read_all(stream_client, ref) + assert [record.body for record in records] == ["a", "b"] + assert offsets == [record.offset.hex() for record in records] + + +@workflow.defn +class ReadAcrossAnOutageWorkflow: + def __init__(self) -> None: + self._bodies: list[str] = [] + + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + async for record in stream.iter_read_items(wait=timedelta(seconds=10)): + self._bodies.append(record.body) + return self._bodies + + @workflow.query + def bodies(self) -> list[str]: + return self._bodies + + +@_needs_redis +async def test_read_across_a_short_outage_gives_each_record_once(client: Client): + assert _REDIS_URL + async with _StoreOutage(_REDIS_URL) as outage: + stream_client = _redis_client(client, outage.url) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + stream = stream_client.get_stream(ref, body_type=str) + async with new_worker(stream_client, ReadAcrossAnOutageWorkflow) as worker: + handle = await stream_client.start_workflow( + ReadAcrossAnOutageWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + + async def read_three() -> None: + bodies = await handle.query(ReadAcrossAnOutageWorkflow.bodies) + assert bodies == ["a", "b", "c"] + + await assert_eventually(read_three) + # The outage cuts the read that waits for the next Record. + outage.take_down() + await asyncio.sleep(1.5) + # Only the Worker's read asks the Store in the outage. + assert outage.refused > 0 + outage.bring_up() + await assert_eventually(lambda: _store_answers(stream)) + await stream.append("d", "e") + await stream.close() + result = await asyncio.wait_for(handle.result(), timeout=30) + assert result == ["a", "b", "c", "d", "e"] + + +@_needs_redis +@pytest.mark.parametrize("opt_in", ["worker", "defn"]) +async def test_replay_with_its_store_down_fails_only_the_task( + client: Client, opt_in: str +): + assert _REDIS_URL + workflow_type: type[Any] = ( + ReadThenWaitFailsWorkflow if opt_in == "defn" else ReadThenWaitWorkflow + ) + worker_options: dict[str, Any] = ( + {"workflow_failure_exception_types": [StreamReplayError]} + if opt_in == "worker" + else {} + ) + async with _StoreOutage(_REDIS_URL) as outage: + stream_client = _redis_client(client, outage.url) + ref = await _filled_stream(stream_client, ["a", "b"]) + task_queue = f"tq-{uuid.uuid4()}" + # No cache, so the next Worker replays the History and reads the Store. + async with new_worker( + stream_client, workflow_type, task_queue=task_queue, max_cached_workflows=0 + ): + handle: WorkflowHandle[Any, Any] = await stream_client.start_workflow( + workflow_type.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=task_queue, + ) + + async def read_is_in_history() -> None: + assert "core_stream" in _markers(await handle.fetch_history()) + + await assert_eventually(read_is_in_history) + outage.take_down() + async with new_worker( + stream_client, workflow_type, task_queue=task_queue, **worker_options + ): + await handle.signal("proceed") + + async def task_failed() -> None: + messages = [ + event.workflow_task_failed_event_attributes.failure.message + for event in (await handle.fetch_history()).events + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_FAILED + ] + assert messages + assert messages[0].startswith( + f"StreamReplayUnavailable: Stream `{ref.id}` in Store `redis`" + ), messages[0] + + # Core tries the Store for 5 s before it fails the task. + await assert_eventually(task_failed, timeout=timedelta(seconds=30)) + # The opt-in doesn't fail the Workflow, since the Records may still be there. + description = await handle.describe() + assert description.status == WorkflowExecutionStatus.RUNNING + outage.bring_up() + result = await asyncio.wait_for(handle.result(), timeout=60) + assert result == ["a", "b"] From 7125279fc0971a7a13a882aaaf733ec81dd64d13 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Tue, 17 Nov 2026 02:53:10 -0700 Subject: [PATCH 57/59] Documented the retry of a Store outage for Direct calls. Core tries a Direct call again while its Store is unavailable. A replay whose Store stays down fails only its Workflow Task, so `StreamReplayError` says it is not that error. --- changelog/added/patient-river-heron.md | 1 + temporalio/client/_stream_store.py | 9 ++++++--- temporalio/contrib/streams/README.md | 10 +++++++++- temporalio/exceptions.py | 11 ++++++++++- temporalio/worker/_replayer.py | 3 ++- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md index b46b3f147..df66750e2 100644 --- a/changelog/added/patient-river-heron.md +++ b/changelog/added/patient-river-heron.md @@ -5,3 +5,4 @@ Core records metrics for the Direct calls of a Stream, such as `temporal_stream_ Add `StreamConfig.workflow_read_ahead_bytes`, the most bytes of one read's Records that wait in the Worker for Workflow code, 4 MiB by default. Add `StreamConfig.workflow_read_batch_window`, the least time between two batches that one read gives to Workflow code, 50 ms by default. Add `temporalio.exceptions.StreamReplayError`. With it in `workflow_failure_exception_types`, a replay that can't read the Records of a read from the Store again fails the Workflow, and not only the Workflow Task. +Core tries a read, an append or a close of Workflow code again while its Store is unavailable, from 100 ms up to 2 s apart, for at most 30 s. Only then does Workflow code get the retryable `StreamError`. A replay tries its Store for 5 s. If the Store stays down, only the Workflow Task fails, with `StreamReplayUnavailable`, also with `StreamReplayError` in `workflow_failure_exception_types`. diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index 0859bed32..f0824cfd2 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -54,8 +54,10 @@ class StreamReplaySource(enum.Enum): Store again and checks the count and the digest. So the replay needs the Stream config and the Records. The Records must stay in the Store, and Retention must keep them, for as long as the Workflow can replay. When they can't be read, the Workflow Task fails - with a message that starts with ``StreamReplayError``. A replay on a Worker with no - cache reads the Store again on every Workflow Task. + with a message that starts with ``StreamReplayError``. A Store that stays unavailable + for 5 s fails the task with a message that starts with ``StreamReplayUnavailable`` + in its place. A replay on a Worker with no cache reads the Store again on every + Workflow Task. """ @@ -66,7 +68,8 @@ class StreamStoreCallOptions: Workflow code can't do I/O, so the default :py:meth:`StreamStore.call_from_workflow` calls its Store from a Local Activity. Core makes a read, and a Workflow's append and close, on a Store with a Rust runtime - itself, with no Activity, so these options don't apply to them. + itself, with no Activity, so these options don't apply to them. Core tries such a + call again while its Store is unavailable, for at most 30 s. A :py:class:`StreamConfig` holds them in :py:attr:`StreamConfig.workflow_call_options`, and every Store of that config diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 78f632e73..6d45e5929 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -101,7 +101,9 @@ read has three other costs. 10 s. A call with a `wait` on a quiet Stream holds its task open for the whole wait. - By default, `StreamReplaySource.STORE`, the marker holds a range of offsets, a count and a digest, and no Record. A replay reads the Store again, so the Records must stay in the - Store for as long as the Workflow can replay. The option + Store for as long as the Workflow can replay. A replay whose Store stays unavailable for + 5 s fails only its Workflow Task, with `StreamReplayUnavailable`, and never the Workflow. + The option `workflow_read_replay_from=StreamReplaySource.HISTORY` of `StreamConfig` puts the Records into the marker, so a replay needs no Store. Then one Workflow Task's marker holds at most 1 MiB of Records. Past that, the task completes and the read goes on in the next one. @@ -118,6 +120,12 @@ their epoch. An append runs inside its Workflow Task and records a marker with n append is at-least-once: when a Worker crashes or its run is evicted before the task with the append completes, the rerun of that task writes a second copy with the epoch of the new task. +Core tries a read, an append or a close again while its Store answers `StorageUnavailable`. +It waits from 100 ms up to 2 s between tries, for at most 30 s. Only then does Workflow code +get the retryable `StreamError`. Every other reason raises on the first try. An append of +Workflow code always carries a `sequence`, so the Store writes its Records once, also when a +try reached the Store but its answer got lost. + A close spans two Workflow Tasks. The task that calls `close` records only the intent. Core closes the Stream after the server took that task, and the next task gives the result. So a failed task closes nothing, and its retry can still append. A Workflow that awaits its close diff --git a/temporalio/exceptions.py b/temporalio/exceptions.py index 5d2380cb4..450543b47 100644 --- a/temporalio/exceptions.py +++ b/temporalio/exceptions.py @@ -218,7 +218,10 @@ class StreamError(ApplicationError): ``STREAM_ERROR_REASON_CLOSED`` (the Stream is closed), or ``STREAM_ERROR_REASON_ALREADY_EXISTS`` (a strict create of a Stream that exists). A Store outage, ``STREAM_ERROR_REASON_STORAGE_UNAVAILABLE``, is the one retryable - reason. + reason. On a Store with a Rust runtime, Core tries a read, an append or a close of + Workflow code again while the Store is unavailable. It waits from 100 ms up to 2 s + between tries, and Workflow code gets this reason only after 30 s. Every other + reason raises on the first try. .. warning:: This API is experimental and subject to change. @@ -330,6 +333,12 @@ class StreamReplayError(TemporalError): instead. Its failure is an :py:class:`ApplicationError` with the ``type`` ``StreamReplayError``, and Core raises no instance of this class. + A Store that doesn't answer is not this error, since the Records may still be + there. A replay tries the Store again for 5 s. If it stays unavailable, only the + Workflow Task fails, with a message that starts with ``StreamReplayUnavailable``. + That holds also with this class in ``workflow_failure_exception_types`` or + ``failure_exception_types``, so a later attempt of the task can read the Records. + .. warning:: This API is experimental and subject to change. """ diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index 9e92b0314..b6620b0a7 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -84,7 +84,8 @@ def __init__( the history. Core reads them from the Store again, so the replay needs this config and Stores that still have the Records. Without them the replay fails with a :py:class:`RuntimeError` whose message has - ``StreamReplayError``, the Stream and the cause. + ``StreamReplayError``, the Stream and the cause. A Store that stays unavailable + for 5 s gives ``StreamReplayUnavailable`` in place of ``StreamReplayError``. Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that From dd5af8aceff351c64b9a7425d936e13e134f0407 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Tue, 17 Nov 2026 12:21:15 -0700 Subject: [PATCH 58/59] Moved Core to Direct reads with no batch window. Core dropped the batch window and gives Workflow code at most 256 Records in one page. So `StreamConfig.workflow_read_batch_window` and its tests are gone, and the docs give the cap. --- changelog/added/patient-river-heron.md | 3 +- temporalio/bridge/sdk-core | 2 +- temporalio/bridge/src/worker.rs | 5 -- temporalio/bridge/worker.py | 2 - temporalio/client/_stream_store.py | 24 ++------ temporalio/contrib/streams/README.md | 9 +-- temporalio/worker/_replayer.py | 4 -- temporalio/worker/_stream_store.py | 11 +--- temporalio/worker/_worker.py | 4 -- tests/nexus/test_stream_direct.py | 78 -------------------------- 10 files changed, 12 insertions(+), 130 deletions(-) diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md index df66750e2..57854266d 100644 --- a/changelog/added/patient-river-heron.md +++ b/changelog/added/patient-river-heron.md @@ -1,8 +1,7 @@ -Add `read`, `iter_read` and `iter_read_items` to the Workflow `Stream`, so Workflow code reads a Stream with the same calls and pages as a client. Core reads the Store inside the Workflow Task. A loop that asks for its next page in the same activation keeps one read open, and a read that no code waits on closes before the activation completes, so it never holds a Workflow Task open. With a `wait`, a quiet Stream gives empty pages, so a loop on a Stream that can be quiet for long uses `iter_read` and leaves after an empty page. A call that waits on a quiet Stream holds its Workflow Task open, with one Workflow Task about every 8 s. The calls of one Workflow Task write one marker in History. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). With `HISTORY`, one Workflow Task's marker holds at most 1 MiB of Records, and the read goes on in the next task. +Add `read`, `iter_read` and `iter_read_items` to the Workflow `Stream`, so Workflow code reads a Stream with the same calls as a client. Core reads the Store inside the Workflow Task. A page that Workflow code gets in one activation has at most 256 Records, and the rest come in the next activation with no wait. A loop that asks for its next page in the same activation keeps one read open, and a read that no code waits on closes before the activation completes, so it never holds a Workflow Task open. With a `wait`, a quiet Stream gives empty pages, so a loop on a Stream that can be quiet for long uses `iter_read` and leaves after an empty page. A call that waits on a quiet Stream holds its Workflow Task open, with one Workflow Task about every 8 s. The calls of one Workflow Task write one marker in History. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). With `HISTORY`, one Workflow Task's marker holds at most 1 MiB of Records, and the read goes on in the next task. A Workflow's append on the Redis Store or `MemoryStreamStore` now runs in Core inside the Workflow Task, with no Local Activity. It writes as the Workflow, with its Workflow Task as the epoch. A Workflow's close on the Redis Store or `MemoryStreamStore` now runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close: a graceful shutdown of the Worker waits at most 10 s for a close that the code doesn't await, and a Worker that dies when the Workflow ends loses it. An append and a close of Workflow code always carry the Workflow Task as their epoch. An append is at-least-once: after a Worker crash or an eviction, the rerun of its task can write a second copy with the epoch of the new task. Core records metrics for the Direct calls of a Stream, such as `temporal_stream_read_total`, `temporal_stream_read_records` and `temporal_stream_close_latency`, each with the attribute `stream_store`, and `temporal_stream_close_abandoned` for a close that a graceful shutdown gave up. Add `StreamConfig.workflow_read_ahead_bytes`, the most bytes of one read's Records that wait in the Worker for Workflow code, 4 MiB by default. -Add `StreamConfig.workflow_read_batch_window`, the least time between two batches that one read gives to Workflow code, 50 ms by default. Add `temporalio.exceptions.StreamReplayError`. With it in `workflow_failure_exception_types`, a replay that can't read the Records of a read from the Store again fails the Workflow, and not only the Workflow Task. Core tries a read, an append or a close of Workflow code again while its Store is unavailable, from 100 ms up to 2 s apart, for at most 30 s. Only then does Workflow code get the retryable `StreamError`. A replay tries its Store for 5 s. If the Store stays down, only the Workflow Task fails, with `StreamReplayUnavailable`, also with `StreamReplayError` in `workflow_failure_exception_types`. diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 39315d79f..3bec8fc0e 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 39315d79f3fcb540b0bfe6ae8ab6284c4f83fde6 +Subproject commit 3bec8fc0eb42f47f2b250f21773448491a921afc diff --git a/temporalio/bridge/src/worker.rs b/temporalio/bridge/src/worker.rs index a2ec86a24..48e3b77c3 100644 --- a/temporalio/bridge/src/worker.rs +++ b/temporalio/bridge/src/worker.rs @@ -73,7 +73,6 @@ pub struct WorkerConfig { /// default. stream_read_ahead_bytes: Option, /// The least time between two batches of one read. `None` keeps Core's default. - stream_read_batch_window_millis: Option, /// Whether a replay that lost the Records of a read fails the Workflow, for every type or /// for the types listed. stream_replay_as_workflow_fail: bool, @@ -744,10 +743,6 @@ fn convert_worker_config( Some(bytes) => stream_runtimes.with_read_ahead_bytes(bytes), None => stream_runtimes, }; - let stream_runtimes = match conf.stream_read_batch_window_millis { - Some(millis) => stream_runtimes.with_read_batch_window(Duration::from_millis(millis)), - None => stream_runtimes, - }; let mut workflow_failure_errors = HashSet::new(); if conf.nondeterminism_as_workflow_fail { workflow_failure_errors.insert(WorkflowErrorType::Nondeterminism); diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index a03b6f224..8b14a0559 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -66,8 +66,6 @@ class WorkerConfig: """The bridge's object of each Store's Stream runtime, by Store name.""" stream_read_ahead_bytes: int | None = None """The most bytes of one read's Records that wait for Workflow code, or Core's default.""" - stream_read_batch_window_millis: int | None = None - """The least time between two batches of one read, or ``None`` for Core's default.""" stream_replay_as_workflow_fail: bool = False """Whether a replay that lost the Records of a read fails the Workflow.""" stream_replay_as_workflow_fail_for_types: set[str] = dataclasses.field( diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index f0824cfd2..aaadeeb1d 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -500,8 +500,9 @@ class StreamConfig: to it. Workflow code reads with the same ``read``, ``iter_read`` and ``iter_read_items`` as - a client, and gets the same pages. On a Store with a Rust runtime, Core reads the - Store inside the Workflow Task. A loop that asks for its next page in the same + a client. On a Store with a Rust runtime, Core reads the Store inside the Workflow + Task. A page that Workflow code gets in one activation has at most 256 Records, and + the rest come in the next activation with no wait. A loop that asks for its next page in the same activation keeps one read open in Core. A read that no code waits on closes before the activation completes, so it never holds a Workflow Task open. @@ -600,17 +601,6 @@ class StreamConfig: for a Worker with little memory, since each open read can hold this much. """ - workflow_read_batch_window: timedelta | None = None - """The least time between two batches that one read gives to Workflow code. - - A read of Workflow code gives its first batch immediately. After that it - gives Workflow code a batch at most once in this window, with every Record that came - in the window. So a busy Stream doesn't activate the Workflow for each Record. A Record - waits at most one window. ``None`` takes Core's default of 50 ms, and zero is no - window. Set a longer window when a busy Stream activates the Workflow too often, and - zero when each Record must reach Workflow code as soon as it comes. - """ - _store_map: dict[str, StreamStore] = dataclasses.field( init=False, repr=False, compare=False ) @@ -621,19 +611,13 @@ def __post_init__(self) -> None: Raises :py:class:`temporalio.exceptions.StreamConfigError` if there is no Store, or if two Stores share a name, and ``ValueError`` if - :py:attr:`workflow_read_ahead_bytes` is below 1 or - :py:attr:`workflow_read_batch_window` is negative. + :py:attr:`workflow_read_ahead_bytes` is below 1. """ if ( self.workflow_read_ahead_bytes is not None and self.workflow_read_ahead_bytes < 1 ): raise ValueError("workflow_read_ahead_bytes must be at least 1") - if ( - self.workflow_read_batch_window is not None - and self.workflow_read_batch_window < timedelta() - ): - raise ValueError("workflow_read_batch_window must not be negative") if not self.stores: raise StreamConfigError("StreamConfig.stores must have at least one Store") store_map: dict[str, StreamStore] = {} diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 6d45e5929..273cf57fb 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -48,7 +48,8 @@ replacement of the first attempt's Records. The helpers run on the Streams API, so they follow how Workflow code reads, appends and closes. On a Store with a Rust runtime, such as the Redis Store or `MemoryStreamStore`, Core reads, appends and closes a Stream itself, with no Activity. Workflow code reads with the same -`read`, `iter_read` and `iter_read_items` as a client, and gets the same pages. +`read`, `iter_read` and `iter_read_items` as a client. It gets the same pages, but each page has +at most 256 Records. Here the Workflow reads the events of a model's answer while the model Activity still runs, and starts each tool as its call arrives. @@ -111,9 +112,9 @@ read has three other costs. While Workflow code is busy, a read holds at most 4 MiB of its Records in the Worker, then waits until the code takes them. `workflow_read_ahead_bytes` of `StreamConfig` changes that -bound, for large Records or a Worker with little memory. A read gives Workflow code a batch at -most once in 50 ms, so a busy Stream doesn't activate the Workflow for each Record. -`workflow_read_batch_window` of `StreamConfig` changes that window. +bound, for large Records or a Worker with little memory. A read gives Workflow code its Records +as soon as no activation of the Workflow is pending. A page that Workflow code gets in one +activation has at most 256 Records, and the rest come in the next activation with no wait. An append and a close of Workflow code always write as the Workflow, with the Workflow Task as their epoch. An append runs inside its Workflow Task and records a marker with no body. An diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index b6620b0a7..513fe2abb 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -24,7 +24,6 @@ from ._interceptor import Interceptor from ._stream_store import ( _bridge_read_ahead_bytes, - _bridge_read_batch_window_millis, _bridge_stream_runtimes, _WorkflowStreamStore, ) @@ -393,9 +392,6 @@ def on_eviction_hook( stream_read_ahead_bytes=_bridge_read_ahead_bytes( self._config.get("stream_config") ), - stream_read_batch_window_millis=_bridge_read_batch_window_millis( - self._config.get("stream_config") - ), ), ) bridge_worker_scope = bridge_worker diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index 6c2d182b1..13881eb9b 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -196,7 +196,7 @@ def _start_replayed_direct_call( # pyright: ignore[reportUnusedFunction] # A client's read gives at most this many Records in a page, and a `max_records` of zero -# or less gives this many. A read of Workflow code gives the same pages. +# or less gives this many. A read of Workflow code takes the same limits. _MAX_READ_RECORDS = 1000 # The longest `wait` of a client's read. A longer one waits this long. _MAX_READ_WAIT = timedelta(seconds=60) @@ -372,15 +372,6 @@ def _bridge_read_ahead_bytes(config: StreamConfig | None) -> int | None: # pyri return config.workflow_read_ahead_bytes if config is not None else None -def _bridge_read_batch_window_millis( # pyright: ignore[reportUnusedFunction] - config: StreamConfig | None, -) -> int | None: - """Give Core the batch window of the config in milliseconds, or ``None`` for Core's default.""" - if config is None or config.workflow_read_batch_window is None: - return None - return round(config.workflow_read_batch_window.total_seconds() * 1000) - - def _bridge_stream_runtimes(config: StreamConfig | None) -> list[tuple[str, Any]]: # pyright: ignore[reportUnusedFunction] """Give Core the bridge's object of each Store's Stream runtime, by Store name.""" if config is None: diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index 78ced650c..a05ed0586 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -36,7 +36,6 @@ from ._plugin import Plugin from ._stream_store import ( _bridge_read_ahead_bytes, - _bridge_read_batch_window_millis, _bridge_stream_runtimes, _WorkflowStreamStore, ) @@ -742,9 +741,6 @@ def check_activity(activity: str): stream_config if self._workflow_worker is not None else None ), stream_read_ahead_bytes=_bridge_read_ahead_bytes(stream_config), - stream_read_batch_window_millis=_bridge_read_batch_window_millis( - stream_config - ), ), ) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index c4c53e4e8..1b9e55b89 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -2375,84 +2375,6 @@ async def test_read_with_a_read_ahead_of_one_byte_gives_every_record(client: Cli assert result.dones[-1] is True -# The batch window. - - -def _read_page_counts(history: WorkflowHistory) -> list[int]: - """Give the Records of each batch of the read entries of the markers, in order.""" - counts: list[int] = [] - for event in history.events: - attributes = event.marker_recorded_event_attributes - if ( - event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED - or attributes.marker_name != "core_stream" - ): - continue - [payload] = attributes.details["data"].payloads - for entry in StreamMarkerData.FromString(payload.data).reads: - counts.extend(entry.page_counts) - return counts - - -@activity.defn -async def append_five_apart(stream_id: str) -> None: - stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) - for index in range(5): - if index: - await asyncio.sleep(0.05) - await stream.append(f"record-{index}") - await stream.close() - - -@workflow.defn -class WindowedReadWorkflow: - @workflow.run - async def run(self, stream_id: str) -> list[str]: - # A Local Activity holds the Workflow Task open, so each batch of the read comes - # in an activation of this task and not in a later task. - writer = workflow.start_local_activity( - append_five_apart, stream_id, start_to_close_timeout=timedelta(seconds=30) - ) - stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) - got = [ - record.body - async for record in stream.iter_read_items(wait=timedelta(seconds=10)) - ] - await writer - return got - - -async def _windowed_read(client: Client, window: timedelta) -> list[int]: - config = client.config() - config["stream_config"] = StreamConfig( - stores=[MemoryStreamStore()], workflow_read_batch_window=window - ) - stream_client = Client(**config) - ref = await _filled_stream(stream_client, [], close=False) - handle, got = await _run( - stream_client, WindowedReadWorkflow, ref.id, activities=[append_five_apart] - ) - assert got == [f"record-{index}" for index in range(5)] - return _read_page_counts(await handle.fetch_history()) - - -async def test_batch_window_joins_the_records_of_a_window(client: Client): - # The first batch comes immediately, and the window holds the other four for 1 s. - assert await _windowed_read(client, timedelta(seconds=1)) == [1, 4] - - -async def test_zero_batch_window_gives_more_batches(client: Client): - assert len(await _windowed_read(client, timedelta())) > 2 - - -def test_negative_batch_window_raises(): - with pytest.raises(ValueError, match="workflow_read_batch_window"): - StreamConfig( - stores=[MemoryStreamStore()], - workflow_read_batch_window=timedelta(milliseconds=-1), - ) - - # A replay that lost the Records of a read. From b6df792ca1c6feeef18f4b52cfb56c945b09ec5b Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Tue, 17 Nov 2026 12:21:20 -0700 Subject: [PATCH 59/59] Tested the 256 Record page cap and the refused long offset. A Store page of 600 Records reaches Workflow code in pages of at most 256 Records. A `from_offset` over 1024 bytes raises `StreamError` with `INVALID`, and the Workflow goes on. --- tests/nexus/test_stream_direct.py | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py index 1b9e55b89..52a21d08a 100644 --- a/tests/nexus/test_stream_direct.py +++ b/tests/nexus/test_stream_direct.py @@ -922,6 +922,32 @@ async def test_read_takes_the_limits_of_a_client_read(client: Client): assert results[0] == ["STREAM_ERROR_REASON_INVALID", expected[0][1]] +@workflow.defn +class LongOffsetWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + got: list[str] = [] + try: + await stream.read(from_offset=b"o" * 2000) + got.append("page") + except StreamError as err: + got.append(StreamErrorReason.Name(err.reason)) + got.extend([record.body async for record in stream.iter_read_items()]) + return got + + +async def test_read_from_an_offset_over_1024_bytes_raises_invalid(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b"]) + handle, got = await _run(stream_client, LongOffsetWorkflow, ref) + assert got == ["STREAM_ERROR_REASON_INVALID", "a", "b"] + config = stream_client.config()["stream_config"] + await Replayer( + workflows=[LongOffsetWorkflow], stream_config=config + ).replay_workflow(history=await handle.fetch_history()) + + @workflow.defn class QueryReadWorkflow: def __init__(self) -> None: @@ -2375,6 +2401,39 @@ async def test_read_with_a_read_ahead_of_one_byte_gives_every_record(client: Cli assert result.dones[-1] is True +# The cap on the Records of one page that Workflow code gets. + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_store_page_over_256_records_comes_in_pages_of_at_most_256( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + bodies = [f"record-{index}" for index in range(600)] + ref = await _filled_stream(stream_client, bodies) + handle, got = await _run( + stream_client, + ReadItemsWorkflow, + ref, + None, + max_cached_workflows=max_cached_workflows, + ) + assert got == bodies + history = await handle.fetch_history() + counts = [ + count + for task in _read_entries(history) + for entry in task + for count in entry.page_counts + ] + assert sum(counts) == 600 + assert max(counts) == 256 + config = stream_client.config()["stream_config"] + await Replayer(workflows=[ReadItemsWorkflow], stream_config=config).replay_workflow( + history + ) + + # A replay that lost the Records of a read.