diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c60bb3316..72aec058b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -228,8 +228,9 @@ jobs: - run: uv tool install poethepoet - run: uv sync --all-extras - run: cargo install --locked nexgen --version 0.2.8 --features advanced --force - # The Stream package needs the `main` of nexgen, which no release has yet. - - run: cargo install --locked --git https://github.com/temporalio/nexgen --rev 9653a0f5a845df0c5059554f1b5813db6a74cd1a --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen + # The Stream package needs `@nexus.helper` with `sync=true`, which only a nexgen fork has. + # This commit of the fork also has the poll iterators of the Workflow target. + - run: cargo install --locked --git https://github.com/moetemp/nexgen --rev 2785462ecdaba5da7a27051fbaaccbaee26ae9e0 --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen - run: echo "NEX_GEN_STREAM_BIN=$RUNNER_TEMP/nexgen-stream/bin/nexgen" >> "$GITHUB_ENV" - run: poe build-develop - run: poe gen-protos diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md new file mode 100644 index 000000000..57854266d --- /dev/null +++ b/changelog/added/patient-river-heron.md @@ -0,0 +1,7 @@ +Add `read`, `iter_read` and `iter_read_items` to the Workflow `Stream`, so Workflow code reads a Stream with the same calls as a client. Core reads the Store inside the Workflow Task. A page that Workflow code gets in one activation has at most 256 Records, and the rest come in the next activation with no wait. A loop that asks for its next page in the same activation keeps one read open, and a read that no code waits on closes before the activation completes, so it never holds a Workflow Task open. With a `wait`, a quiet Stream gives empty pages, so a loop on a Stream that can be quiet for long uses `iter_read` and leaves after an empty page. A call that waits on a quiet Stream holds its Workflow Task open, with one Workflow Task about every 8 s. The calls of one Workflow Task write one marker in History. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). With `HISTORY`, one Workflow Task's marker holds at most 1 MiB of Records, and the read goes on in the next task. +A Workflow's append on the Redis Store or `MemoryStreamStore` now runs in Core inside the Workflow Task, with no Local Activity. It writes as the Workflow, with its Workflow Task as the epoch. +A Workflow's close on the Redis Store or `MemoryStreamStore` now runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close: a graceful shutdown of the Worker waits at most 10 s for a close that the code doesn't await, and a Worker that dies when the Workflow ends loses it. An append and a close of Workflow code always carry the Workflow Task as their epoch. An append is at-least-once: after a Worker crash or an eviction, the rerun of its task can write a second copy with the epoch of the new task. +Core records metrics for the Direct calls of a Stream, such as `temporal_stream_read_total`, `temporal_stream_read_records` and `temporal_stream_close_latency`, each with the attribute `stream_store`, and `temporal_stream_close_abandoned` for a close that a graceful shutdown gave up. +Add `StreamConfig.workflow_read_ahead_bytes`, the most bytes of one read's Records that wait in the Worker for Workflow code, 4 MiB by default. +Add `temporalio.exceptions.StreamReplayError`. With it in `workflow_failure_exception_types`, a replay that can't read the Records of a read from the Store again fails the Workflow, and not only the Workflow Task. +Core tries a read, an append or a close of Workflow code again while its Store is unavailable, from 100 ms up to 2 s apart, for at most 30 s. Only then does Workflow code get the retryable `StreamError`. A replay tries its Store for 5 s. If the Store stays down, only the Workflow Task fails, with `StreamReplayUnavailable`, also with `StreamReplayError` in `workflow_failure_exception_types`. diff --git a/scripts/gen_nexus_system_api.py b/scripts/gen_nexus_system_api.py index 48998fc7c..b62c423ef 100644 --- a/scripts/gen_nexus_system_api.py +++ b/scripts/gen_nexus_system_api.py @@ -28,8 +28,10 @@ workflow_init_path = base_dir / "temporalio" / "workflow" / "__init__.py" NEX_GEN_VERSION = "0.2.8" # The Stream package needs a newer nexgen, which rejects the other WITs. One pin is enough -# again once the api WITs and a nexgen release agree. No release has nexgen commit 9653a0f5, -# the `main` of nexgen, yet, so NEX_GEN_STREAM_BIN points at a build of it. +# again once the api WITs and a nexgen release agree. The Stream WIT uses `@nexus.helper` with +# `sync=true` and `@nexus.support-type`, which only the fork https://github.com/moetemp/nexgen +# has. Its commit 2785462e also has the poll iterators of the Workflow target. So +# NEX_GEN_STREAM_BIN points at a build of that commit. NEX_GEN_STREAM_VERSION = "0.2.12" proto_files = [ gen_protos.api_proto_dir @@ -87,7 +89,8 @@ def nex_gen_stream_command() -> list[str]: bin_path = os.environ.get("NEX_GEN_STREAM_BIN") if not bin_path: raise RuntimeError( - "NEX_GEN_STREAM_BIN must point at a nexgen build of commit 9653a0f5" + "NEX_GEN_STREAM_BIN must point at a nexgen build of commit 2785462e " + "of https://github.com/moetemp/nexgen" ) version = subprocess.check_output([bin_path, "--version"], text=True).strip() if version != f"nexgen {NEX_GEN_STREAM_VERSION}": diff --git a/temporalio/api/stream/v1/message_pb2.pyi b/temporalio/api/stream/v1/message_pb2.pyi index 1d0103602..1e42e74a2 100644 --- a/temporalio/api/stream/v1/message_pb2.pyi +++ b/temporalio/api/stream/v1/message_pb2.pyi @@ -167,8 +167,8 @@ global___ActivityEpoch = ActivityEpoch @typing.final class WorkflowProducer(google.protobuf.message.Message): """Experimental. Might change in the future. - A Workflow run Producer. On the server's Streams, a Workflow writes as this Producer in its - Workflow Task. On a Store, Workflow code writes through a Local Activity, so its writes carry a + A Workflow run Producer. Workflow code that writes in its Workflow Task writes as this + Producer, with that task as the epoch. A write that it makes through a Local Activity has a Local Activity Producer. """ @@ -180,7 +180,9 @@ class WorkflowProducer(google.protobuf.message.Message): """Required. The run id of the Workflow run, not of its chain. Each run is its own Producer.""" @property def epoch(self) -> global___WorkflowTaskEpoch: - """The Workflow Task of the write.""" + """The Workflow Task of the write. Workflow code leaves it empty, and the SDK sets it before + the write goes to the Store. + """ def __init__( self, diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index 064e7b369..200959871 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -2927,6 +2927,7 @@ dependencies = [ "temporalio-client", "temporalio-common", "temporalio-macros", + "temporalio-streams", "thiserror", "tokio", "tokio-stream", diff --git a/temporalio/bridge/_visitor.py b/temporalio/bridge/_visitor.py index 8832b56bc..54238f891 100644 --- a/temporalio/bridge/_visitor.py +++ b/temporalio/bridge/_visitor.py @@ -308,6 +308,38 @@ async def _visit_coresdk_workflow_activation_ResolveNexusOperation( if o.HasField("result"): await self._visit_coresdk_nexus_NexusOperationResult(fs, o.result) + async def _visit_temporal_api_stream_v1_StreamRecord( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("body"): + await self._visit_temporal_api_common_v1_Payload(fs, o.body) + + async def _visit_coresdk_workflow_activation_StreamReadEnd( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("failure"): + await self._visit_temporal_api_failure_v1_Failure(fs, o.failure) + + async def _visit_coresdk_workflow_activation_ResolveStreamRead( + self, fs: VisitorFunctions, o: Any + ): + for v in o.records: + await self._visit_temporal_api_stream_v1_StreamRecord(fs, v) + if o.HasField("end"): + await self._visit_coresdk_workflow_activation_StreamReadEnd(fs, o.end) + + async def _visit_coresdk_workflow_activation_ResolveStreamAppend( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("failed"): + await self._visit_temporal_api_failure_v1_Failure(fs, o.failed) + + async def _visit_coresdk_workflow_activation_ResolveStreamClose( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("failed"): + await self._visit_temporal_api_failure_v1_Failure(fs, o.failed) + async def _visit_coresdk_workflow_activation_WorkflowActivationJob( self, fs: VisitorFunctions, o: Any ): @@ -353,6 +385,18 @@ async def _visit_coresdk_workflow_activation_WorkflowActivationJob( await self._visit_coresdk_workflow_activation_ResolveNexusOperation( fs, o.resolve_nexus_operation ) + elif o.HasField("resolve_stream_read"): + await self._visit_coresdk_workflow_activation_ResolveStreamRead( + fs, o.resolve_stream_read + ) + elif o.HasField("resolve_stream_append"): + await self._visit_coresdk_workflow_activation_ResolveStreamAppend( + fs, o.resolve_stream_append + ) + elif o.HasField("resolve_stream_close"): + await self._visit_coresdk_workflow_activation_ResolveStreamClose( + fs, o.resolve_stream_close + ) async def _visit_coresdk_workflow_activation_WorkflowActivation( self, fs: VisitorFunctions, o: Any @@ -492,6 +536,11 @@ async def _visit_coresdk_workflow_commands_ScheduleNexusOperation( if o.HasField("input"): await self._visit_nexus_operation_input_payload(fs, o.input) + async def _visit_coresdk_workflow_commands_AppendToStream( + self, fs: VisitorFunctions, o: Any + ): + await self._visit_payload_container(fs, o.bodies) + async def _visit_coresdk_workflow_commands_WorkflowCommand( self, fs: VisitorFunctions, o: Any ): @@ -551,6 +600,10 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand( await self._visit_coresdk_workflow_commands_ScheduleNexusOperation( fs, o.schedule_nexus_operation ) + elif o.HasField("append_to_stream"): + await self._visit_coresdk_workflow_commands_AppendToStream( + fs, o.append_to_stream + ) async def _visit_coresdk_workflow_completion_Success( self, fs: VisitorFunctions, o: Any @@ -681,12 +734,6 @@ async def _visit_temporal_api_streamservice_v1_ReadRecordsRequest( ) -> None: pass - async def _visit_temporal_api_stream_v1_StreamRecord( - self, fs: VisitorFunctions, o: Any - ): - if o.HasField("body"): - await self._visit_temporal_api_common_v1_Payload(fs, o.body) - async def _visit_temporal_api_streamservice_v1_ReadRecordsResponse( self, fs: VisitorFunctions, o: Any ): diff --git a/temporalio/bridge/proto/common/__init__.py b/temporalio/bridge/proto/common/__init__.py index a8506090d..b9b0cce02 100644 --- a/temporalio/bridge/proto/common/__init__.py +++ b/temporalio/bridge/proto/common/__init__.py @@ -1,6 +1,7 @@ from .common_pb2 import ( ExternalStorageMetrics, NamespacedWorkflowExecution, + StreamReadStop, VersioningIntent, WorkerDeploymentVersion, ) @@ -8,6 +9,7 @@ __all__ = [ "ExternalStorageMetrics", "NamespacedWorkflowExecution", + "StreamReadStop", "VersioningIntent", "WorkerDeploymentVersion", ] diff --git a/temporalio/bridge/proto/common/common_pb2.py b/temporalio/bridge/proto/common/common_pb2.py index 0b9a5c816..e492e93c1 100644 --- a/temporalio/bridge/proto/common/common_pb2.py +++ b/temporalio/bridge/proto/common/common_pb2.py @@ -16,7 +16,7 @@ from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n%temporal/sdk/core/common/common.proto\x12\x0e\x63oresdk.common\x1a\x1egoogle/protobuf/duration.proto"U\n\x1bNamespacedWorkflowExecution\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"D\n\x17WorkerDeploymentVersion\x12\x17\n\x0f\x64\x65ployment_name\x18\x01 \x01(\t\x12\x10\n\x08\x62uild_id\x18\x02 \x01(\t"\x92\x01\n\x16\x45xternalStorageMetrics\x12\x15\n\rpayload_count\x18\x01 \x01(\x04\x12\x18\n\x10total_size_bytes\x18\x02 \x01(\x04\x12\x31\n\x0etotal_duration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x14\n\x0c\x64river_names\x18\x04 \x03(\t*@\n\x10VersioningIntent\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCOMPATIBLE\x10\x01\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x02\x42,\xea\x02)Temporalio::Internal::Bridge::Api::Commonb\x06proto3' + b'\n%temporal/sdk/core/common/common.proto\x12\x0e\x63oresdk.common\x1a\x1egoogle/protobuf/duration.proto"U\n\x1bNamespacedWorkflowExecution\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"D\n\x17WorkerDeploymentVersion\x12\x17\n\x0f\x64\x65ployment_name\x18\x01 \x01(\t\x12\x10\n\x08\x62uild_id\x18\x02 \x01(\t"\x92\x01\n\x16\x45xternalStorageMetrics\x12\x15\n\rpayload_count\x18\x01 \x01(\x04\x12\x18\n\x10total_size_bytes\x18\x02 \x01(\x04\x12\x31\n\x0etotal_duration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x14\n\x0c\x64river_names\x18\x04 \x03(\t*\xc8\x01\n\x0eStreamReadStop\x12 \n\x1cSTREAM_READ_STOP_UNSPECIFIED\x10\x00\x12\x19\n\x15STREAM_READ_STOP_DONE\x10\x01\x12 \n\x1cSTREAM_READ_STOP_MAX_RECORDS\x10\x02\x12\x1e\n\x1aSTREAM_READ_STOP_MAX_BYTES\x10\x03\x12\x19\n\x15STREAM_READ_STOP_IDLE\x10\x04\x12\x1c\n\x18STREAM_READ_STOP_TIMEOUT\x10\x05*@\n\x10VersioningIntent\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCOMPATIBLE\x10\x01\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x02\x42,\xea\x02)Temporalio::Internal::Bridge::Api::Commonb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -26,8 +26,10 @@ DESCRIPTOR._serialized_options = ( b"\352\002)Temporalio::Internal::Bridge::Api::Common" ) - _VERSIONINGINTENT._serialized_start = 395 - _VERSIONINGINTENT._serialized_end = 459 + _STREAMREADSTOP._serialized_start = 396 + _STREAMREADSTOP._serialized_end = 596 + _VERSIONINGINTENT._serialized_start = 598 + _VERSIONINGINTENT._serialized_end = 662 _NAMESPACEDWORKFLOWEXECUTION._serialized_start = 89 _NAMESPACEDWORKFLOWEXECUTION._serialized_end = 174 _WORKERDEPLOYMENTVERSION._serialized_start = 176 diff --git a/temporalio/bridge/proto/common/common_pb2.pyi b/temporalio/bridge/proto/common/common_pb2.pyi index 7c2b05702..da85e098d 100644 --- a/temporalio/bridge/proto/common/common_pb2.pyi +++ b/temporalio/bridge/proto/common/common_pb2.pyi @@ -21,6 +21,45 @@ else: DESCRIPTOR: google.protobuf.descriptor.FileDescriptor +class _StreamReadStop: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _StreamReadStopEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + _StreamReadStop.ValueType + ], + builtins.type, +): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + STREAM_READ_STOP_UNSPECIFIED: _StreamReadStop.ValueType # 0 + STREAM_READ_STOP_DONE: _StreamReadStop.ValueType # 1 + """The Stream is closed, and the read gave its last Record.""" + STREAM_READ_STOP_MAX_RECORDS: _StreamReadStop.ValueType # 2 + """The read gave `max_records` Records.""" + STREAM_READ_STOP_MAX_BYTES: _StreamReadStop.ValueType # 3 + """The next Record would pass `max_bytes`.""" + STREAM_READ_STOP_IDLE: _StreamReadStop.ValueType # 4 + """No Record came for `idle_timeout`.""" + STREAM_READ_STOP_TIMEOUT: _StreamReadStop.ValueType # 5 + """`timeout` passed since the read started.""" + +class StreamReadStop(_StreamReadStop, metaclass=_StreamReadStopEnumTypeWrapper): + """Why a read of a Stream ended. A read that failed has no stop.""" + +STREAM_READ_STOP_UNSPECIFIED: StreamReadStop.ValueType # 0 +STREAM_READ_STOP_DONE: StreamReadStop.ValueType # 1 +"""The Stream is closed, and the read gave its last Record.""" +STREAM_READ_STOP_MAX_RECORDS: StreamReadStop.ValueType # 2 +"""The read gave `max_records` Records.""" +STREAM_READ_STOP_MAX_BYTES: StreamReadStop.ValueType # 3 +"""The next Record would pass `max_bytes`.""" +STREAM_READ_STOP_IDLE: StreamReadStop.ValueType # 4 +"""No Record came for `idle_timeout`.""" +STREAM_READ_STOP_TIMEOUT: StreamReadStop.ValueType # 5 +"""`timeout` passed since the read started.""" +global___StreamReadStop = StreamReadStop + class _VersioningIntent: ValueType = typing.NewType("ValueType", builtins.int) V: typing_extensions.TypeAlias = ValueType diff --git a/temporalio/bridge/proto/external_data/__init__.py b/temporalio/bridge/proto/external_data/__init__.py index 836e97ef4..ffe40588e 100644 --- a/temporalio/bridge/proto/external_data/__init__.py +++ b/temporalio/bridge/proto/external_data/__init__.py @@ -1,6 +1,19 @@ -from .external_data_pb2 import LocalActivityMarkerData, PatchedMarkerData +from .external_data_pb2 import ( + LocalActivityMarkerData, + PatchedMarkerData, + StreamAppendEntry, + StreamCloseEntry, + StreamMarkerData, + StreamReadEnded, + StreamReadEntry, +) __all__ = [ "LocalActivityMarkerData", "PatchedMarkerData", + "StreamAppendEntry", + "StreamCloseEntry", + "StreamMarkerData", + "StreamReadEnded", + "StreamReadEntry", ] diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.py b/temporalio/bridge/proto/external_data/external_data_pb2.py index 4bda6f514..3e13e28c1 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.py +++ b/temporalio/bridge/proto/external_data/external_data_pb2.py @@ -16,8 +16,18 @@ from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from temporalio.api.failure.v1 import ( + message_pb2 as temporal_dot_api_dot_failure_dot_v1_dot_message__pb2, +) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) +from temporalio.bridge.proto.common import ( + common_pb2 as temporal_dot_sdk_dot_core_dot_common_dot_common__pb2, +) + DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\xb2\x02\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x10\x61\x63tivation_index\x18\x08 \x01(\x04H\x00\x88\x01\x01\x42\x13\n\x11_activation_index"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08\x42\x32\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' + b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a%temporal/sdk/core/common/common.proto"\xb2\x02\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x10\x61\x63tivation_index\x18\x08 \x01(\x04H\x00\x88\x01\x01\x42\x13\n\x11_activation_index"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xf1\x01\n\x10StreamMarkerData\x12\x35\n\x05reads\x18\x01 \x03(\x0b\x32&.coresdk.external_data.StreamReadEntry\x12\x39\n\x07\x61ppends\x18\x02 \x03(\x0b\x32(.coresdk.external_data.StreamAppendEntry\x12\x37\n\x06\x63loses\x18\x03 \x03(\x0b\x32\'.coresdk.external_data.StreamCloseEntry\x12\x32\n\x07streams\x18\x04 \x03(\x0b\x32!.temporal.api.stream.v1.StreamRef"\xae\x02\n\x0fStreamReadEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x14\n\x0cstream_index\x18\x02 \x01(\r\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12\x14\n\x0c\x61\x66ter_offset\x18\x04 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x05 \x01(\x0c\x12\r\n\x05\x63ount\x18\x06 \x01(\r\x12\x0e\n\x06\x64igest\x18\x07 \x01(\x0c\x12\x35\n\x07records\x18\x08 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x13\n\x0bpage_counts\x18\t \x03(\r\x12\x18\n\x10page_activations\x18\n \x03(\r\x12\x33\n\x03\x65nd\x18\x0b \x01(\x0b\x32&.coresdk.external_data.StreamReadEnded"\xa9\x01\n\x0fStreamReadEnded\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x05 \x01(\r"\xa8\x01\n\x11StreamAppendEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x14\n\x0cstream_index\x18\x02 \x01(\r\x12\x14\n\x0c\x66irst_offset\x18\x03 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x04 \x01(\x0c\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\r"\xcf\x01\n\x10StreamCloseEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x14\n\x0cstream_index\x18\x02 \x01(\r\x12?\n\x0cintent_epoch\x18\x03 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch\x12\x10\n\x08resolved\x18\x04 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\rB2\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -27,8 +37,18 @@ DESCRIPTOR._serialized_options = ( b"\352\002/Temporalio::Internal::Bridge::Api::ExternalData" ) - _LOCALACTIVITYMARKERDATA._serialized_start = 144 - _LOCALACTIVITYMARKERDATA._serialized_end = 450 - _PATCHEDMARKERDATA._serialized_start = 452 - _PATCHEDMARKERDATA._serialized_end = 503 + _LOCALACTIVITYMARKERDATA._serialized_start = 260 + _LOCALACTIVITYMARKERDATA._serialized_end = 566 + _PATCHEDMARKERDATA._serialized_start = 568 + _PATCHEDMARKERDATA._serialized_end = 619 + _STREAMMARKERDATA._serialized_start = 622 + _STREAMMARKERDATA._serialized_end = 863 + _STREAMREADENTRY._serialized_start = 866 + _STREAMREADENTRY._serialized_end = 1168 + _STREAMREADENDED._serialized_start = 1171 + _STREAMREADENDED._serialized_end = 1340 + _STREAMAPPENDENTRY._serialized_start = 1343 + _STREAMAPPENDENTRY._serialized_end = 1511 + _STREAMCLOSEENTRY._serialized_start = 1514 + _STREAMCLOSEENTRY._serialized_end = 1721 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.pyi b/temporalio/bridge/proto/external_data/external_data_pb2.pyi index edcd4581a..61b500977 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.pyi +++ b/temporalio/bridge/proto/external_data/external_data_pb2.pyi @@ -4,13 +4,19 @@ isort:skip_file """ import builtins +import collections.abc import typing import google.protobuf.descriptor import google.protobuf.duration_pb2 +import google.protobuf.internal.containers import google.protobuf.message import google.protobuf.timestamp_pb2 +import temporalio.api.failure.v1.message_pb2 +import temporalio.api.stream.v1.message_pb2 +import temporalio.bridge.proto.common.common_pb2 + DESCRIPTOR: google.protobuf.descriptor.FileDescriptor @typing.final @@ -139,3 +145,340 @@ class PatchedMarkerData(google.protobuf.message.Message): ) -> None: ... global___PatchedMarkerData = PatchedMarkerData + +@typing.final +class StreamMarkerData(google.protobuf.message.Message): + """The data of the marker `core_stream`, which Core records once for each workflow task in which + the workflow got a Stream result. It is in the detail `data` as `binary/protobuf`, so that a + marker with Records stays small. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + READS_FIELD_NUMBER: builtins.int + APPENDS_FIELD_NUMBER: builtins.int + CLOSES_FIELD_NUMBER: builtins.int + STREAMS_FIELD_NUMBER: builtins.int + @property + def reads( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamReadEntry + ]: ... + @property + def appends( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamAppendEntry + ]: ... + @property + def closes( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamCloseEntry + ]: ... + @property + def streams( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRef + ]: + """Each Stream of the entries, once. An entry gives the index of its Stream here, so that a task + with many results on one Stream holds its ref once. + """ + + def __init__( + self, + *, + reads: collections.abc.Iterable[global___StreamReadEntry] | None = ..., + appends: collections.abc.Iterable[global___StreamAppendEntry] | None = ..., + closes: collections.abc.Iterable[global___StreamCloseEntry] | None = ..., + streams: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRef + ] + | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing.Literal[ + "appends", + b"appends", + "closes", + b"closes", + "reads", + b"reads", + "streams", + b"streams", + ], + ) -> None: ... + +global___StreamMarkerData = StreamMarkerData + +@typing.final +class StreamReadEntry(google.protobuf.message.Message): + """What one read gave the workflow in one workflow task.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_INDEX_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + AFTER_OFFSET_FIELD_NUMBER: builtins.int + LAST_OFFSET_FIELD_NUMBER: builtins.int + COUNT_FIELD_NUMBER: builtins.int + DIGEST_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + PAGE_COUNTS_FIELD_NUMBER: builtins.int + PAGE_ACTIVATIONS_FIELD_NUMBER: builtins.int + END_FIELD_NUMBER: builtins.int + seq: builtins.int + stream_index: builtins.int + """The index of the read's Stream in `StreamMarkerData.streams`.""" + from_offset: builtins.bytes + """The `from_offset` of the read, so a replay can check that the read is the same.""" + after_offset: builtins.bytes + """The offset before the first Record of this task. A replay reads after it.""" + last_offset: builtins.bytes + """The offset of the last Record of this task. Empty when the task got no Record.""" + count: builtins.int + """The number of Records of this task.""" + digest: builtins.bytes + """The digest of the Records of this task. Empty when `records` holds them.""" + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: + """The Records of this task, only for a read with its Records in history.""" + + @property + def page_counts( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: + """The number of Records of each page, in order. A page ends where its count ends.""" + + @property + def page_activations( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: + """The activation of each page in the task. The first is the index, and each later one is the + gap to the one before. + """ + + @property + def end(self) -> global___StreamReadEnded: + """Set when the read ended in this task.""" + + def __init__( + self, + *, + seq: builtins.int = ..., + stream_index: builtins.int = ..., + from_offset: builtins.bytes = ..., + after_offset: builtins.bytes = ..., + last_offset: builtins.bytes = ..., + count: builtins.int = ..., + digest: builtins.bytes = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + page_counts: collections.abc.Iterable[builtins.int] | None = ..., + page_activations: collections.abc.Iterable[builtins.int] | None = ..., + end: global___StreamReadEnded | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["end", b"end"]) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "after_offset", + b"after_offset", + "count", + b"count", + "digest", + b"digest", + "end", + b"end", + "from_offset", + b"from_offset", + "last_offset", + b"last_offset", + "page_activations", + b"page_activations", + "page_counts", + b"page_counts", + "records", + b"records", + "seq", + b"seq", + "stream_index", + b"stream_index", + ], + ) -> None: ... + +global___StreamReadEntry = StreamReadEntry + +@typing.final +class StreamReadEnded(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LAST_OFFSET_FIELD_NUMBER: builtins.int + STOP_FIELD_NUMBER: builtins.int + DONE_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + ACTIVATION_FIELD_NUMBER: builtins.int + last_offset: builtins.bytes + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType + done: builtins.bool + activation: builtins.int + """The activation in the task that got the end.""" + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + last_offset: builtins.bytes = ..., + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType = ..., + done: builtins.bool = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + activation: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failure", b"failure"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "activation", + b"activation", + "done", + b"done", + "failure", + b"failure", + "last_offset", + b"last_offset", + "stop", + b"stop", + ], + ) -> None: ... + +global___StreamReadEnded = StreamReadEnded + +@typing.final +class StreamAppendEntry(google.protobuf.message.Message): + """The result of one append, in the workflow task whose activation got it.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_INDEX_FIELD_NUMBER: builtins.int + FIRST_OFFSET_FIELD_NUMBER: builtins.int + LAST_OFFSET_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + ACTIVATION_FIELD_NUMBER: builtins.int + seq: builtins.int + stream_index: builtins.int + """The index of the append's Stream in `StreamMarkerData.streams`.""" + first_offset: builtins.bytes + last_offset: builtins.bytes + activation: builtins.int + """The activation in the task that got the result.""" + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + seq: builtins.int = ..., + stream_index: builtins.int = ..., + first_offset: builtins.bytes = ..., + last_offset: builtins.bytes = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + activation: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failure", b"failure"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "activation", + b"activation", + "failure", + b"failure", + "first_offset", + b"first_offset", + "last_offset", + b"last_offset", + "seq", + b"seq", + "stream_index", + b"stream_index", + ], + ) -> None: ... + +global___StreamAppendEntry = StreamAppendEntry + +@typing.final +class StreamCloseEntry(google.protobuf.message.Message): + """A close in one workflow task: its intent, in the task that asked for it, or its result, in the + task whose activation got it. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_INDEX_FIELD_NUMBER: builtins.int + INTENT_EPOCH_FIELD_NUMBER: builtins.int + RESOLVED_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + ACTIVATION_FIELD_NUMBER: builtins.int + seq: builtins.int + stream_index: builtins.int + """The index of the closed Stream in `StreamMarkerData.streams`.""" + resolved: builtins.bool + """True on the result. `failure` is unset when the Stream is closed.""" + activation: builtins.int + """The activation in the task that got the result.""" + @property + def intent_epoch(self) -> temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch: + """Set on the intent, the epoch that Core closes with after the server took the task.""" + + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + seq: builtins.int = ..., + stream_index: builtins.int = ..., + intent_epoch: temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch + | None = ..., + resolved: builtins.bool = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + activation: builtins.int = ..., + ) -> None: ... + def HasField( + self, + field_name: typing.Literal[ + "failure", b"failure", "intent_epoch", b"intent_epoch" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "activation", + b"activation", + "failure", + b"failure", + "intent_epoch", + b"intent_epoch", + "resolved", + b"resolved", + "seq", + b"seq", + "stream_index", + b"stream_index", + ], + ) -> None: ... + +global___StreamCloseEntry = StreamCloseEntry diff --git a/temporalio/bridge/proto/workflow_activation/__init__.py b/temporalio/bridge/proto/workflow_activation/__init__.py index 5c178e218..4e459cc9b 100644 --- a/temporalio/bridge/proto/workflow_activation/__init__.py +++ b/temporalio/bridge/proto/workflow_activation/__init__.py @@ -16,7 +16,12 @@ ResolveNexusOperationStart, ResolveRequestCancelExternalWorkflow, ResolveSignalExternalWorkflow, + ResolveStreamAppend, + ResolveStreamClose, + ResolveStreamRead, SignalWorkflow, + StreamAppended, + StreamReadEnd, UpdateRandomSeed, WorkflowActivation, WorkflowActivationJob, @@ -40,7 +45,12 @@ "ResolveNexusOperationStart", "ResolveRequestCancelExternalWorkflow", "ResolveSignalExternalWorkflow", + "ResolveStreamAppend", + "ResolveStreamClose", + "ResolveStreamRead", "SignalWorkflow", + "StreamAppended", + "StreamReadEnd", "UpdateRandomSeed", "WorkflowActivation", "WorkflowActivationJob", diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py index 589119c72..863fbc082 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py @@ -32,6 +32,9 @@ from temporalio.api.nexus.v1 import ( message_pb2 as temporal_dot_api_dot_nexus_dot_v1_dot_message__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.api.update.v1 import ( message_pb2 as temporal_dot_api_dot_update_dot_v1_dot_message__pb2, ) @@ -49,7 +52,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\xe0\n\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\xe3\x0b\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x12\x65\n&propagated_nexus_serialization_context\x18\x1b \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"\xb1\x01\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause"\xb8\x01\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause"\xb2\x03\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x12\x65\n&propagated_nexus_serialization_context\x18\x08 \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\nB8\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' + b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\xd3\x0c\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12M\n\x13resolve_stream_read\x18\x11 \x01(\x0b\x32..coresdk.workflow_activation.ResolveStreamReadH\x00\x12Q\n\x15resolve_stream_append\x18\x12 \x01(\x0b\x32\x30.coresdk.workflow_activation.ResolveStreamAppendH\x00\x12O\n\x14resolve_stream_close\x18\x13 \x01(\x0b\x32/.coresdk.workflow_activation.ResolveStreamCloseH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\xe3\x0b\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x12\x65\n&propagated_nexus_serialization_context\x18\x1b \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"\xb1\x01\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause"\xb8\x01\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause"\xb2\x03\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x12\x65\n&propagated_nexus_serialization_context\x18\x08 \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\n"\x90\x01\n\x11ResolveStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x35\n\x07records\x18\x02 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x37\n\x03\x65nd\x18\x03 \x01(\x0b\x32*.coresdk.workflow_activation.StreamReadEnd"\x93\x01\n\rStreamReadEnd\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\xa1\x01\n\x13ResolveStreamAppend\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12?\n\x08\x61ppended\x18\x02 \x01(\x0b\x32+.coresdk.workflow_activation.StreamAppendedH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status";\n\x0eStreamAppended\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x02 \x01(\x0c"S\n\x12ResolveStreamClose\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x30\n\x06\x66\x61iled\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.FailureB8\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -67,56 +70,66 @@ _SIGNALWORKFLOW_HEADERSENTRY._serialized_options = b"8\001" _DOUPDATE_HEADERSENTRY._options = None _DOUPDATE_HEADERSENTRY._serialized_options = b"8\001" - _WORKFLOWACTIVATION._serialized_start = 611 - _WORKFLOWACTIVATION._serialized_end = 1157 - _WORKFLOWACTIVATIONJOB._serialized_start = 1160 - _WORKFLOWACTIVATIONJOB._serialized_end = 2536 - _INITIALIZEWORKFLOW._serialized_start = 2539 - _INITIALIZEWORKFLOW._serialized_end = 4046 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 3967 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 4046 - _FIRETIMER._serialized_start = 4048 - _FIRETIMER._serialized_end = 4072 - _RESOLVEACTIVITY._serialized_start = 4074 - _RESOLVEACTIVITY._serialized_end = 4183 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4186 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4523 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4525 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4584 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4587 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 4753 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 4755 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 4851 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 4853 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 4958 - _UPDATERANDOMSEED._serialized_start = 4960 - _UPDATERANDOMSEED._serialized_end = 5003 - _QUERYWORKFLOW._serialized_start = 5006 - _QUERYWORKFLOW._serialized_end = 5266 - _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 3967 - _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 4046 - _CANCELWORKFLOW._serialized_start = 5268 - _CANCELWORKFLOW._serialized_end = 5300 - _SIGNALWORKFLOW._serialized_start = 5303 - _SIGNALWORKFLOW._serialized_end = 5592 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 3967 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 4046 - _NOTIFYHASPATCH._serialized_start = 5594 - _NOTIFYHASPATCH._serialized_end = 5628 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5631 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 5808 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 5811 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 5995 - _DOUPDATE._serialized_start = 5998 - _DOUPDATE._serialized_end = 6432 - _DOUPDATE_HEADERSENTRY._serialized_start = 3967 - _DOUPDATE_HEADERSENTRY._serialized_end = 4046 - _RESOLVENEXUSOPERATIONSTART._serialized_start = 6435 - _RESOLVENEXUSOPERATIONSTART._serialized_end = 6589 - _RESOLVENEXUSOPERATION._serialized_start = 6591 - _RESOLVENEXUSOPERATION._serialized_end = 6680 - _REMOVEFROMCACHE._serialized_start = 6683 - _REMOVEFROMCACHE._serialized_end = 7035 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 6797 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 7035 + _WORKFLOWACTIVATION._serialized_start = 649 + _WORKFLOWACTIVATION._serialized_end = 1195 + _WORKFLOWACTIVATIONJOB._serialized_start = 1198 + _WORKFLOWACTIVATIONJOB._serialized_end = 2817 + _INITIALIZEWORKFLOW._serialized_start = 2820 + _INITIALIZEWORKFLOW._serialized_end = 4327 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 4248 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 4327 + _FIRETIMER._serialized_start = 4329 + _FIRETIMER._serialized_end = 4353 + _RESOLVEACTIVITY._serialized_start = 4355 + _RESOLVEACTIVITY._serialized_end = 4464 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4467 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4804 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4806 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4865 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4868 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 5034 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 5036 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 5132 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 5134 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 5239 + _UPDATERANDOMSEED._serialized_start = 5241 + _UPDATERANDOMSEED._serialized_end = 5284 + _QUERYWORKFLOW._serialized_start = 5287 + _QUERYWORKFLOW._serialized_end = 5547 + _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 4248 + _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 4327 + _CANCELWORKFLOW._serialized_start = 5549 + _CANCELWORKFLOW._serialized_end = 5581 + _SIGNALWORKFLOW._serialized_start = 5584 + _SIGNALWORKFLOW._serialized_end = 5873 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 4248 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 4327 + _NOTIFYHASPATCH._serialized_start = 5875 + _NOTIFYHASPATCH._serialized_end = 5909 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5912 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 6089 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 6092 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 6276 + _DOUPDATE._serialized_start = 6279 + _DOUPDATE._serialized_end = 6713 + _DOUPDATE_HEADERSENTRY._serialized_start = 4248 + _DOUPDATE_HEADERSENTRY._serialized_end = 4327 + _RESOLVENEXUSOPERATIONSTART._serialized_start = 6716 + _RESOLVENEXUSOPERATIONSTART._serialized_end = 6870 + _RESOLVENEXUSOPERATION._serialized_start = 6872 + _RESOLVENEXUSOPERATION._serialized_end = 6961 + _REMOVEFROMCACHE._serialized_start = 6964 + _REMOVEFROMCACHE._serialized_end = 7316 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 7078 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 7316 + _RESOLVESTREAMREAD._serialized_start = 7319 + _RESOLVESTREAMREAD._serialized_end = 7463 + _STREAMREADEND._serialized_start = 7466 + _STREAMREADEND._serialized_end = 7613 + _RESOLVESTREAMAPPEND._serialized_start = 7616 + _RESOLVESTREAMAPPEND._serialized_end = 7777 + _STREAMAPPENDED._serialized_start = 7779 + _STREAMAPPENDED._serialized_end = 7838 + _RESOLVESTREAMCLOSE._serialized_start = 7840 + _RESOLVESTREAMCLOSE._serialized_end = 7923 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi index af1a2e38a..de21dd362 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi @@ -23,6 +23,7 @@ import temporalio.api.enums.v1.failed_cause_pb2 import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 import temporalio.api.nexus.v1.message_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.update.v1.message_pb2 import temporalio.bridge.proto.activity_result.activity_result_pb2 import temporalio.bridge.proto.child_workflow.child_workflow_pb2 @@ -244,6 +245,9 @@ class WorkflowActivationJob(google.protobuf.message.Message): DO_UPDATE_FIELD_NUMBER: builtins.int RESOLVE_NEXUS_OPERATION_START_FIELD_NUMBER: builtins.int RESOLVE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int + RESOLVE_STREAM_READ_FIELD_NUMBER: builtins.int + RESOLVE_STREAM_APPEND_FIELD_NUMBER: builtins.int + RESOLVE_STREAM_CLOSE_FIELD_NUMBER: builtins.int REMOVE_FROM_CACHE_FIELD_NUMBER: builtins.int @property def initialize_workflow(self) -> global___InitializeWorkflow: @@ -318,6 +322,18 @@ class WorkflowActivationJob(google.protobuf.message.Message): def resolve_nexus_operation(self) -> global___ResolveNexusOperation: """A nexus operation resolved.""" + @property + def resolve_stream_read(self) -> global___ResolveStreamRead: + """A read of a Stream gave Records, ended, or both.""" + + @property + def resolve_stream_append(self) -> global___ResolveStreamAppend: + """An append to a Stream resolved.""" + + @property + def resolve_stream_close(self) -> global___ResolveStreamClose: + """A close of a Stream resolved.""" + @property def remove_from_cache(self) -> global___RemoveFromCache: """Remove the workflow identified by the [WorkflowActivation] containing this job from the @@ -347,6 +363,9 @@ class WorkflowActivationJob(google.protobuf.message.Message): do_update: global___DoUpdate | None = ..., resolve_nexus_operation_start: global___ResolveNexusOperationStart | None = ..., resolve_nexus_operation: global___ResolveNexusOperation | None = ..., + resolve_stream_read: global___ResolveStreamRead | None = ..., + resolve_stream_append: global___ResolveStreamAppend | None = ..., + resolve_stream_close: global___ResolveStreamClose | None = ..., remove_from_cache: global___RemoveFromCache | None = ..., ) -> None: ... def HasField( @@ -380,6 +399,12 @@ class WorkflowActivationJob(google.protobuf.message.Message): b"resolve_request_cancel_external_workflow", "resolve_signal_external_workflow", b"resolve_signal_external_workflow", + "resolve_stream_append", + b"resolve_stream_append", + "resolve_stream_close", + b"resolve_stream_close", + "resolve_stream_read", + b"resolve_stream_read", "signal_workflow", b"signal_workflow", "update_random_seed", @@ -419,6 +444,12 @@ class WorkflowActivationJob(google.protobuf.message.Message): b"resolve_request_cancel_external_workflow", "resolve_signal_external_workflow", b"resolve_signal_external_workflow", + "resolve_stream_append", + b"resolve_stream_append", + "resolve_stream_close", + b"resolve_stream_close", + "resolve_stream_read", + b"resolve_stream_read", "signal_workflow", b"signal_workflow", "update_random_seed", @@ -446,6 +477,9 @@ class WorkflowActivationJob(google.protobuf.message.Message): "do_update", "resolve_nexus_operation_start", "resolve_nexus_operation", + "resolve_stream_read", + "resolve_stream_append", + "resolve_stream_close", "remove_from_cache", ] | None @@ -1639,3 +1673,186 @@ class RemoveFromCache(google.protobuf.message.Message): ) -> None: ... global___RemoveFromCache = RemoveFromCache + +@typing.final +class ResolveStreamRead(google.protobuf.message.Message): + """The Records that a read of a Stream gave since its last job, and its end if it ended.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + END_FIELD_NUMBER: builtins.int + seq: builtins.int + """The sequence number of the `StartStreamRead`""" + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: ... + @property + def end(self) -> global___StreamReadEnd: + """Set on the last job of a read.""" + + def __init__( + self, + *, + seq: builtins.int = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + end: global___StreamReadEnd | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["end", b"end"]) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal["end", b"end", "records", b"records", "seq", b"seq"], + ) -> None: ... + +global___ResolveStreamRead = ResolveStreamRead + +@typing.final +class StreamReadEnd(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LAST_OFFSET_FIELD_NUMBER: builtins.int + STOP_FIELD_NUMBER: builtins.int + DONE_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + last_offset: builtins.bytes + """The offset of the last Record that the workflow got, or `from_offset` if it got none.""" + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType + """Why the read ended. Unspecified when it failed.""" + done: builtins.bool + """True if the Stream is closed and the read gave its last Record.""" + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: + """Set if the read failed.""" + + def __init__( + self, + *, + last_offset: builtins.bytes = ..., + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType = ..., + done: builtins.bool = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failure", b"failure"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "done", + b"done", + "failure", + b"failure", + "last_offset", + b"last_offset", + "stop", + b"stop", + ], + ) -> None: ... + +global___StreamReadEnd = StreamReadEnd + +@typing.final +class ResolveStreamAppend(google.protobuf.message.Message): + """The result of an `AppendToStream`.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + APPENDED_FIELD_NUMBER: builtins.int + FAILED_FIELD_NUMBER: builtins.int + seq: builtins.int + """The sequence number of the `AppendToStream`""" + @property + def appended(self) -> global___StreamAppended: ... + @property + def failed(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + seq: builtins.int = ..., + appended: global___StreamAppended | None = ..., + failed: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing.Literal[ + "appended", b"appended", "failed", b"failed", "status", b"status" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "appended", + b"appended", + "failed", + b"failed", + "seq", + b"seq", + "status", + b"status", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing.Literal["status", b"status"] + ) -> typing.Literal["appended", "failed"] | None: ... + +global___ResolveStreamAppend = ResolveStreamAppend + +@typing.final +class StreamAppended(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FIRST_OFFSET_FIELD_NUMBER: builtins.int + LAST_OFFSET_FIELD_NUMBER: builtins.int + first_offset: builtins.bytes + last_offset: builtins.bytes + def __init__( + self, + *, + first_offset: builtins.bytes = ..., + last_offset: builtins.bytes = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing.Literal[ + "first_offset", b"first_offset", "last_offset", b"last_offset" + ], + ) -> None: ... + +global___StreamAppended = StreamAppended + +@typing.final +class ResolveStreamClose(google.protobuf.message.Message): + """The result of a `CloseStream`, in a workflow task after the one that asked for it.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + FAILED_FIELD_NUMBER: builtins.int + seq: builtins.int + """The sequence number of the `CloseStream`""" + @property + def failed(self) -> temporalio.api.failure.v1.message_pb2.Failure: + """Unset when the Stream is closed, also when it was closed already.""" + + def __init__( + self, + *, + seq: builtins.int = ..., + failed: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failed", b"failed"] + ) -> builtins.bool: ... + def ClearField( + self, field_name: typing.Literal["failed", b"failed", "seq", b"seq"] + ) -> None: ... + +global___ResolveStreamClose = ResolveStreamClose diff --git a/temporalio/bridge/proto/workflow_commands/__init__.py b/temporalio/bridge/proto/workflow_commands/__init__.py index f599917af..15dde24ee 100644 --- a/temporalio/bridge/proto/workflow_commands/__init__.py +++ b/temporalio/bridge/proto/workflow_commands/__init__.py @@ -1,9 +1,12 @@ from .workflow_commands_pb2 import ( ActivityCancellationType, + AppendToStream, CancelChildWorkflowExecution, CancelSignalWorkflow, CancelTimer, CancelWorkflowExecution, + CloseStream, + CloseStreamRead, CompleteWorkflowExecution, ContinueAsNewWorkflowExecution, FailWorkflowExecution, @@ -20,6 +23,7 @@ SetPatchMarker, SignalExternalWorkflowExecution, StartChildWorkflowExecution, + StartStreamRead, StartTimer, UpdateResponse, UpsertWorkflowSearchAttributes, @@ -28,10 +32,13 @@ __all__ = [ "ActivityCancellationType", + "AppendToStream", "CancelChildWorkflowExecution", "CancelSignalWorkflow", "CancelTimer", "CancelWorkflowExecution", + "CloseStream", + "CloseStreamRead", "CompleteWorkflowExecution", "ContinueAsNewWorkflowExecution", "FailWorkflowExecution", @@ -48,6 +55,7 @@ "SetPatchMarker", "SignalExternalWorkflowExecution", "StartChildWorkflowExecution", + "StartStreamRead", "StartTimer", "UpdateResponse", "UpsertWorkflowSearchAttributes", diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py index 3573f36b1..1908ea226 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py @@ -32,6 +32,9 @@ from temporalio.api.sdk.v1 import ( user_metadata_pb2 as temporal_dot_api_dot_sdk_dot_v1_dot_user__metadata__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.api.workflow.v1 import ( message_pb2 as temporal_dot_api_dot_workflow_dot_v1_dot_message__pb2, ) @@ -46,7 +49,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\xa9\x10\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x42\t\n\x07variant"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x93\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12#\n\x1binclude_arguments_in_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xe1\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12I\n\x13versioning_override\x18\x15 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' + b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a$temporal/api/stream/v1/message.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\xc2\x12\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x12G\n\x11start_stream_read\x18\x17 \x01(\x0b\x32*.coresdk.workflow_commands.StartStreamReadH\x00\x12G\n\x11\x63lose_stream_read\x18\x18 \x01(\x0b\x32*.coresdk.workflow_commands.CloseStreamReadH\x00\x12\x45\n\x10\x61ppend_to_stream\x18\x19 \x01(\x0b\x32).coresdk.workflow_commands.AppendToStreamH\x00\x12>\n\x0c\x63lose_stream\x18\x1a \x01(\x0b\x32&.coresdk.workflow_commands.CloseStreamH\x00\x42\t\n\x07variant"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x93\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12#\n\x1binclude_arguments_in_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xe1\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12I\n\x13versioning_override\x18\x15 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x87\x02\n\x0fStartStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12*\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12/\n\x0cidle_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bmax_records\x18\x06 \x01(\r\x12\x11\n\tmax_bytes\x18\x07 \x01(\x04\x12\x1a\n\x12records_in_history\x18\x08 \x01(\x08"\x1e\n\x0f\x43loseStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xc7\x01\n\x0e\x41ppendToStream\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12/\n\x06\x62odies\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x32\n\x08producer\x18\x04 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12\x10\n\x08sequence\x18\x05 \x01(\x03"\x81\x01\n\x0b\x43loseStream\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x32\n\x08producer\x18\x03 \x01(\x0b\x32 .temporal.api.stream.v1.Producer*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -72,70 +75,78 @@ _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_options = b"8\001" _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._options = None _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_options = b"8\001" - _ACTIVITYCANCELLATIONTYPE._serialized_start = 8785 - _ACTIVITYCANCELLATIONTYPE._serialized_end = 8873 - _WORKFLOWCOMMAND._serialized_start = 558 - _WORKFLOWCOMMAND._serialized_end = 2647 - _STARTTIMER._serialized_start = 2649 - _STARTTIMER._serialized_end = 2732 - _CANCELTIMER._serialized_start = 2734 - _CANCELTIMER._serialized_end = 2760 - _SCHEDULEACTIVITY._serialized_start = 2763 - _SCHEDULEACTIVITY._serialized_end = 3587 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3508 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3587 - _SCHEDULELOCALACTIVITY._serialized_start = 3590 - _SCHEDULELOCALACTIVITY._serialized_end = 4377 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3508 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3587 - _REQUESTCANCELACTIVITY._serialized_start = 4379 - _REQUESTCANCELACTIVITY._serialized_end = 4415 - _REQUESTCANCELLOCALACTIVITY._serialized_start = 4417 - _REQUESTCANCELLOCALACTIVITY._serialized_end = 4458 - _QUERYRESULT._serialized_start = 4461 - _QUERYRESULT._serialized_end = 4617 - _QUERYSUCCESS._serialized_start = 4619 - _QUERYSUCCESS._serialized_end = 4684 - _COMPLETEWORKFLOWEXECUTION._serialized_start = 4686 - _COMPLETEWORKFLOWEXECUTION._serialized_end = 4762 - _FAILWORKFLOWEXECUTION._serialized_start = 4764 - _FAILWORKFLOWEXECUTION._serialized_end = 4838 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 4841 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 5755 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5598 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5674 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3508 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3587 - _CANCELWORKFLOWEXECUTION._serialized_start = 5757 - _CANCELWORKFLOWEXECUTION._serialized_end = 5833 - _SETPATCHMARKER._serialized_start = 5835 - _SETPATCHMARKER._serialized_end = 5889 - _STARTCHILDWORKFLOWEXECUTION._serialized_start = 5892 - _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7141 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3508 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3587 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5598 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5674 - _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7143 - _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7217 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7220 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7362 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7365 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 7764 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3508 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3587 - _CANCELSIGNALWORKFLOW._serialized_start = 7766 - _CANCELSIGNALWORKFLOW._serialized_end = 7801 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 7803 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 7904 - _MODIFYWORKFLOWPROPERTIES._serialized_start = 7906 - _MODIFYWORKFLOWPROPERTIES._serialized_end = 7985 - _UPDATERESPONSE._serialized_start = 7988 - _UPDATERESPONSE._serialized_end = 8198 - _SCHEDULENEXUSOPERATION._serialized_start = 8201 - _SCHEDULENEXUSOPERATION._serialized_end = 8739 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 8689 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 8739 - _REQUESTCANCELNEXUSOPERATION._serialized_start = 8741 - _REQUESTCANCELNEXUSOPERATION._serialized_end = 8783 + _ACTIVITYCANCELLATIONTYPE._serialized_start = 9736 + _ACTIVITYCANCELLATIONTYPE._serialized_end = 9824 + _WORKFLOWCOMMAND._serialized_start = 596 + _WORKFLOWCOMMAND._serialized_end = 2966 + _STARTTIMER._serialized_start = 2968 + _STARTTIMER._serialized_end = 3051 + _CANCELTIMER._serialized_start = 3053 + _CANCELTIMER._serialized_end = 3079 + _SCHEDULEACTIVITY._serialized_start = 3082 + _SCHEDULEACTIVITY._serialized_end = 3906 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3827 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3906 + _SCHEDULELOCALACTIVITY._serialized_start = 3909 + _SCHEDULELOCALACTIVITY._serialized_end = 4696 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3827 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3906 + _REQUESTCANCELACTIVITY._serialized_start = 4698 + _REQUESTCANCELACTIVITY._serialized_end = 4734 + _REQUESTCANCELLOCALACTIVITY._serialized_start = 4736 + _REQUESTCANCELLOCALACTIVITY._serialized_end = 4777 + _QUERYRESULT._serialized_start = 4780 + _QUERYRESULT._serialized_end = 4936 + _QUERYSUCCESS._serialized_start = 4938 + _QUERYSUCCESS._serialized_end = 5003 + _COMPLETEWORKFLOWEXECUTION._serialized_start = 5005 + _COMPLETEWORKFLOWEXECUTION._serialized_end = 5081 + _FAILWORKFLOWEXECUTION._serialized_start = 5083 + _FAILWORKFLOWEXECUTION._serialized_end = 5157 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 5160 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 6074 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5917 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5993 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3827 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3906 + _CANCELWORKFLOWEXECUTION._serialized_start = 6076 + _CANCELWORKFLOWEXECUTION._serialized_end = 6152 + _SETPATCHMARKER._serialized_start = 6154 + _SETPATCHMARKER._serialized_end = 6208 + _STARTCHILDWORKFLOWEXECUTION._serialized_start = 6211 + _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7460 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3827 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3906 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5917 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5993 + _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7462 + _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7536 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7539 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7681 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7684 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 8083 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3827 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3906 + _CANCELSIGNALWORKFLOW._serialized_start = 8085 + _CANCELSIGNALWORKFLOW._serialized_end = 8120 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 8122 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 8223 + _MODIFYWORKFLOWPROPERTIES._serialized_start = 8225 + _MODIFYWORKFLOWPROPERTIES._serialized_end = 8304 + _UPDATERESPONSE._serialized_start = 8307 + _UPDATERESPONSE._serialized_end = 8517 + _SCHEDULENEXUSOPERATION._serialized_start = 8520 + _SCHEDULENEXUSOPERATION._serialized_end = 9058 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 9008 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 9058 + _REQUESTCANCELNEXUSOPERATION._serialized_start = 9060 + _REQUESTCANCELNEXUSOPERATION._serialized_end = 9102 + _STARTSTREAMREAD._serialized_start = 9105 + _STARTSTREAMREAD._serialized_end = 9368 + _CLOSESTREAMREAD._serialized_start = 9370 + _CLOSESTREAMREAD._serialized_end = 9400 + _APPENDTOSTREAM._serialized_start = 9403 + _APPENDTOSTREAM._serialized_end = 9602 + _CLOSESTREAM._serialized_start = 9605 + _CLOSESTREAM._serialized_end = 9734 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi index 81c47c0b0..233572f0a 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi @@ -25,6 +25,7 @@ import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 import temporalio.api.sdk.v1.event_group_marker_pb2 import temporalio.api.sdk.v1.user_metadata_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.workflow.v1.message_pb2 import temporalio.bridge.proto.child_workflow.child_workflow_pb2 import temporalio.bridge.proto.common.common_pb2 @@ -105,6 +106,10 @@ class WorkflowCommand(google.protobuf.message.Message): UPDATE_RESPONSE_FIELD_NUMBER: builtins.int SCHEDULE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int REQUEST_CANCEL_NEXUS_OPERATION_FIELD_NUMBER: builtins.int + START_STREAM_READ_FIELD_NUMBER: builtins.int + CLOSE_STREAM_READ_FIELD_NUMBER: builtins.int + APPEND_TO_STREAM_FIELD_NUMBER: builtins.int + CLOSE_STREAM_FIELD_NUMBER: builtins.int @property def user_metadata(self) -> temporalio.api.sdk.v1.user_metadata_pb2.UserMetadata: """User metadata that may or may not be persisted into history depending on the command type. @@ -181,6 +186,14 @@ class WorkflowCommand(google.protobuf.message.Message): def request_cancel_nexus_operation( self, ) -> global___RequestCancelNexusOperation: ... + @property + def start_stream_read(self) -> global___StartStreamRead: ... + @property + def close_stream_read(self) -> global___CloseStreamRead: ... + @property + def append_to_stream(self) -> global___AppendToStream: ... + @property + def close_stream(self) -> global___CloseStream: ... def __init__( self, *, @@ -219,10 +232,16 @@ class WorkflowCommand(google.protobuf.message.Message): schedule_nexus_operation: global___ScheduleNexusOperation | None = ..., request_cancel_nexus_operation: global___RequestCancelNexusOperation | None = ..., + start_stream_read: global___StartStreamRead | None = ..., + close_stream_read: global___CloseStreamRead | None = ..., + append_to_stream: global___AppendToStream | None = ..., + close_stream: global___CloseStream | None = ..., ) -> None: ... def HasField( self, field_name: typing.Literal[ + "append_to_stream", + b"append_to_stream", "cancel_child_workflow_execution", b"cancel_child_workflow_execution", "cancel_signal_workflow", @@ -231,6 +250,10 @@ class WorkflowCommand(google.protobuf.message.Message): b"cancel_timer", "cancel_workflow_execution", b"cancel_workflow_execution", + "close_stream", + b"close_stream", + "close_stream_read", + b"close_stream_read", "complete_workflow_execution", b"complete_workflow_execution", "continue_as_new_workflow_execution", @@ -261,6 +284,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"signal_external_workflow_execution", "start_child_workflow_execution", b"start_child_workflow_execution", + "start_stream_read", + b"start_stream_read", "start_timer", b"start_timer", "update_response", @@ -276,6 +301,8 @@ class WorkflowCommand(google.protobuf.message.Message): def ClearField( self, field_name: typing.Literal[ + "append_to_stream", + b"append_to_stream", "cancel_child_workflow_execution", b"cancel_child_workflow_execution", "cancel_signal_workflow", @@ -284,6 +311,10 @@ class WorkflowCommand(google.protobuf.message.Message): b"cancel_timer", "cancel_workflow_execution", b"cancel_workflow_execution", + "close_stream", + b"close_stream", + "close_stream_read", + b"close_stream_read", "complete_workflow_execution", b"complete_workflow_execution", "continue_as_new_workflow_execution", @@ -316,6 +347,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"signal_external_workflow_execution", "start_child_workflow_execution", b"start_child_workflow_execution", + "start_stream_read", + b"start_stream_read", "start_timer", b"start_timer", "update_response", @@ -354,6 +387,10 @@ class WorkflowCommand(google.protobuf.message.Message): "update_response", "schedule_nexus_operation", "request_cancel_nexus_operation", + "start_stream_read", + "close_stream_read", + "append_to_stream", + "close_stream", ] | None ): ... @@ -1931,3 +1968,211 @@ class RequestCancelNexusOperation(google.protobuf.message.Message): def ClearField(self, field_name: typing.Literal["seq", b"seq"]) -> None: ... global___RequestCancelNexusOperation = RequestCancelNexusOperation + +@typing.final +class StartStreamRead(google.protobuf.message.Message): + """Start a read of a Stream. Core reads the Store itself and gives each page of Records to the + workflow in a `ResolveStreamRead` job, until the read ends or the workflow closes it. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + TIMEOUT_FIELD_NUMBER: builtins.int + IDLE_TIMEOUT_FIELD_NUMBER: builtins.int + MAX_RECORDS_FIELD_NUMBER: builtins.int + MAX_BYTES_FIELD_NUMBER: builtins.int + RECORDS_IN_HISTORY_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's incremental sequence number of Stream reads, used as the read's identifier""" + from_offset: builtins.bytes + """The read starts after this offset. Empty means the start of the Stream.""" + max_records: builtins.int + """The most Records the read gives. Zero means no cap.""" + max_bytes: builtins.int + """The most bytes of Records the read gives. The first Record is always given. Zero means no + cap. + """ + records_in_history: builtins.bool + """If set, the Records go into the workflow history, so a replay reads no Store.""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def timeout(self) -> google.protobuf.duration_pb2.Duration: + """The longest time the read may take. Unset means no bound. A set value of zero or less fails + the workflow task. A read that gave no Record yet looks at the Store once before its timeout + ends it. + """ + + @property + def idle_timeout(self) -> google.protobuf.duration_pb2.Duration: + """The read ends when no Record comes for this long. Unset means 1 minute. Zero gives the + Records that are in the Stream, and ends with no wait. A read that must wait longer sets a + longer value. + """ + + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + from_offset: builtins.bytes = ..., + timeout: google.protobuf.duration_pb2.Duration | None = ..., + idle_timeout: google.protobuf.duration_pb2.Duration | None = ..., + max_records: builtins.int = ..., + max_bytes: builtins.int = ..., + records_in_history: builtins.bool = ..., + ) -> None: ... + def HasField( + self, + field_name: typing.Literal[ + "idle_timeout", b"idle_timeout", "stream", b"stream", "timeout", b"timeout" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "from_offset", + b"from_offset", + "idle_timeout", + b"idle_timeout", + "max_bytes", + b"max_bytes", + "max_records", + b"max_records", + "records_in_history", + b"records_in_history", + "seq", + b"seq", + "stream", + b"stream", + "timeout", + b"timeout", + ], + ) -> None: ... + +global___StartStreamRead = StartStreamRead + +@typing.final +class CloseStreamRead(google.protobuf.message.Message): + """Stop a read that `StartStreamRead` started. The workflow gets no job for it.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's sequence number as passed to `StartStreamRead`""" + def __init__( + self, + *, + seq: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["seq", b"seq"]) -> None: ... + +global___CloseStreamRead = CloseStreamRead + +@typing.final +class AppendToStream(google.protobuf.message.Message): + """Append Records to a Stream. Core calls the Store itself and gives the result in a + `ResolveStreamAppend` job. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + BODIES_FIELD_NUMBER: builtins.int + PRODUCER_FIELD_NUMBER: builtins.int + SEQUENCE_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's incremental sequence number of Stream appends, used as the append's identifier""" + sequence: builtins.int + """The `sequence` of the Store request, as the caller gives it. Zero means Core uses `seq`.""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def bodies( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.common.v1.message_pb2.Payload + ]: + """The bodies of the Records, in order""" + + @property + def producer(self) -> temporalio.api.stream.v1.message_pb2.Producer: + """The workflow's Producer, of kind workflow, with its id and run id and no epoch. Core sets the + epoch to the current workflow task. + """ + + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + bodies: collections.abc.Iterable[temporalio.api.common.v1.message_pb2.Payload] + | None = ..., + producer: temporalio.api.stream.v1.message_pb2.Producer | None = ..., + sequence: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["producer", b"producer", "stream", b"stream"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "bodies", + b"bodies", + "producer", + b"producer", + "seq", + b"seq", + "sequence", + b"sequence", + "stream", + b"stream", + ], + ) -> None: ... + +global___AppendToStream = AppendToStream + +@typing.final +class CloseStream(google.protobuf.message.Message): + """Close a Stream. Core records the intent in this workflow task, closes the Stream after the + server took the task, and gives the result in a `ResolveStreamClose` job of a later task. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + PRODUCER_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's incremental sequence number of Stream closes, used as the close's identifier""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def producer(self) -> temporalio.api.stream.v1.message_pb2.Producer: + """The workflow's Producer, of kind workflow, with its id and run id and no epoch. Core sets the + epoch to the workflow task that recorded the close. + """ + + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + producer: temporalio.api.stream.v1.message_pb2.Producer | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["producer", b"producer", "stream", b"stream"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "producer", b"producer", "seq", b"seq", "stream", b"stream" + ], + ) -> None: ... + +global___CloseStream = CloseStream diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 9b121cf32..3bec8fc0e 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 9b121cf32dcd7920a067a04fb61bbb000178290b +Subproject commit 3bec8fc0eb42f47f2b250f21773448491a921afc diff --git a/temporalio/bridge/src/streams.rs b/temporalio/bridge/src/streams.rs index 5eb56cc72..e005a5afc 100644 --- a/temporalio/bridge/src/streams.rs +++ b/temporalio/bridge/src/streams.rs @@ -16,7 +16,7 @@ pyo3::create_exception!(temporal_sdk_bridge, StreamCallError, PyException); #[cfg(feature = "streams")] #[pyclass(frozen)] pub struct StreamRuntimeRef { - streams: std::sync::Arc, + pub(crate) streams: std::sync::Arc, runtime: runtime::Runtime, } diff --git a/temporalio/bridge/src/worker.rs b/temporalio/bridge/src/worker.rs index 39cdc5e77..48e3b77c3 100644 --- a/temporalio/bridge/src/worker.rs +++ b/temporalio/bridge/src/worker.rs @@ -5,7 +5,7 @@ use prost::Message; use pyo3::exceptions::{PyException, PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyTuple}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::sync::{Arc, OnceLock}; use std::time::Duration; @@ -65,6 +65,18 @@ pub struct WorkerConfig { plugins: Vec<(String, String)>, storage_drivers: HashSet, disable_payload_error_limit: bool, + /// The Stream runtime of each Store that has one, by Store name. Core reads and appends a + /// Stream of Workflow code through them. + #[cfg(feature = "streams")] + stream_runtimes: Vec<(String, Py)>, + /// The most bytes of one read's Records that wait for Workflow code. `None` keeps Core's + /// default. + stream_read_ahead_bytes: Option, + /// The least time between two batches of one read. `None` keeps Core's default. + /// Whether a replay that lost the Records of a read fails the Workflow, for every type or + /// for the types listed. + stream_replay_as_workflow_fail: bool, + stream_replay_as_workflow_fail_for_types: HashSet, } #[derive(FromPyObject)] @@ -718,6 +730,40 @@ fn convert_worker_config( ) -> PyResult { let converted_tuner = convert_tuner_holder(conf.tuner, task_locals)?; let converted_versioning_strategy = convert_versioning_strategy(conf.versioning_strategy); + // A build with no Stream runtime has no Store that Core can call, so Core gets none. + #[cfg(feature = "streams")] + let stream_runtimes = temporalio_sdk_core::StreamRuntimes::new( + conf.stream_runtimes + .into_iter() + .map(|(store, runtime)| (store, runtime.get().streams.clone())), + ); + #[cfg(not(feature = "streams"))] + let stream_runtimes = temporalio_sdk_core::StreamRuntimes::default(); + let stream_runtimes = match conf.stream_read_ahead_bytes { + Some(bytes) => stream_runtimes.with_read_ahead_bytes(bytes), + None => stream_runtimes, + }; + let mut workflow_failure_errors = HashSet::new(); + if conf.nondeterminism_as_workflow_fail { + workflow_failure_errors.insert(WorkflowErrorType::Nondeterminism); + } + if conf.stream_replay_as_workflow_fail { + workflow_failure_errors.insert(WorkflowErrorType::StreamReplay); + } + let mut workflow_types_to_failure_errors: HashMap> = + HashMap::new(); + for workflow_type in conf.nondeterminism_as_workflow_fail_for_types { + workflow_types_to_failure_errors + .entry(workflow_type) + .or_default() + .insert(WorkflowErrorType::Nondeterminism); + } + for workflow_type in conf.stream_replay_as_workflow_fail_for_types { + workflow_types_to_failure_errors + .entry(workflow_type) + .or_default() + .insert(WorkflowErrorType::StreamReplay); + } temporalio_sdk_core::WorkerConfig::builder() .namespace(conf.namespace) .task_queue(conf.task_queue) @@ -747,22 +793,8 @@ fn convert_worker_config( // auto-cancel-activity behavior of shutdown will not occur, so we // always set it even if 0. .graceful_shutdown_period(Duration::from_millis(conf.graceful_shutdown_period_millis)) - .workflow_failure_errors(if conf.nondeterminism_as_workflow_fail { - HashSet::from([WorkflowErrorType::Nondeterminism]) - } else { - HashSet::new() - }) - .workflow_types_to_failure_errors( - conf.nondeterminism_as_workflow_fail_for_types - .iter() - .map(|s| { - ( - s.to_owned(), - HashSet::from([WorkflowErrorType::Nondeterminism]), - ) - }) - .collect(), - ) + .workflow_failure_errors(workflow_failure_errors) + .workflow_types_to_failure_errors(workflow_types_to_failure_errors) .maybe_nexus_task_poller_behavior(conf.nexus_task_poller_behavior.map(Into::into)) .plugins( conf.plugins @@ -777,6 +809,7 @@ fn convert_worker_config( .collect::>(), ) .disable_payload_error_limit(conf.disable_payload_error_limit) + .stream_runtimes(stream_runtimes) .build() .map_err(|err| PyValueError::new_err(format!("Invalid worker config: {err}"))) } diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index fa5f337ca..8b14a0559 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -5,9 +5,11 @@ from __future__ import annotations +import dataclasses from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass from typing import ( + Any, TypeAlias, ) @@ -60,6 +62,16 @@ class WorkerConfig: plugins: Sequence[tuple[str, str]] storage_drivers: set[str] disable_payload_error_limit: bool + stream_runtimes: Sequence[tuple[str, Any]] = () + """The bridge's object of each Store's Stream runtime, by Store name.""" + stream_read_ahead_bytes: int | None = None + """The most bytes of one read's Records that wait for Workflow code, or Core's default.""" + stream_replay_as_workflow_fail: bool = False + """Whether a replay that lost the Records of a read fails the Workflow.""" + stream_replay_as_workflow_fail_for_types: set[str] = dataclasses.field( + default_factory=set + ) + """The Workflow types for which such a replay fails the Workflow.""" @dataclass @@ -368,6 +380,7 @@ async def encode_completion( skip_search_attributes=True, skip_headers=not encode_headers, concurrency_limit=storage_concurrency_limit, + skip_stream_bodies=True, ).visit( _Visitor(data_converter._external_store_payload_sequence), completion, diff --git a/temporalio/client/__init__.py b/temporalio/client/__init__.py index 6fa6dd710..8f567ea21 100644 --- a/temporalio/client/__init__.py +++ b/temporalio/client/__init__.py @@ -187,6 +187,7 @@ ) from ._stream_store import ( StreamConfig, + StreamReplaySource, StreamSelectContext, StreamStore, StreamStoreCallOptions, @@ -357,6 +358,7 @@ "CloudOperationsClient", "Plugin", "StreamConfig", + "StreamReplaySource", "StreamSelectContext", "StreamStore", "StreamStoreCallOptions", diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index c2f1364c9..aaadeeb1d 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -1,6 +1,7 @@ from __future__ import annotations import dataclasses +import enum from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable, Sequence from datetime import timedelta @@ -25,12 +26,50 @@ import temporalio.bridge.streams +class StreamReplaySource(enum.Enum): + """Where a replay of a read of Workflow code gets the Records of the read. + + :py:attr:`StreamConfig.workflow_read_replay_from` holds it. It decides what a new read + records in History. A replay follows what History holds, so a change of the setting + doesn't break a Workflow that is running. Client and Activity code have no History, so + the setting doesn't touch them. + + .. warning:: + This API is experimental and subject to change. + """ + + HISTORY = "history" + """The marker in History holds the Records of the read with their bodies. + + A replay takes the Records from History, so it reads no Store and needs no Stream + config. Each Record adds to History. The marker of one Workflow Task holds at most + 1 MiB of Records, and a read goes on in the next task after that. + """ + + STORE = "store" + """The marker in History holds no Record and no body. This is the default. + + For each Workflow Task of a read, the marker holds the range of offsets, the count + and a digest of the Records that the Workflow got. A replay reads the Records from the + Store again and checks the count and the digest. So the replay needs the Stream config + and the Records. The Records must stay in the Store, and Retention must keep them, for + as long as the Workflow can replay. When they can't be read, the Workflow Task fails + with a message that starts with ``StreamReplayError``. A Store that stays unavailable + for 5 s fails the task with a message that starts with ``StreamReplayUnavailable`` + in its place. A replay on a Worker with no cache reads the Store again on every + Workflow Task. + """ + + @dataclasses.dataclass(frozen=True) class StreamStoreCallOptions: """Options of the Local Activity that calls a Stream Store for Workflow code. Workflow code can't do I/O, so the default :py:meth:`StreamStore.call_from_workflow` calls its Store from a Local Activity. + Core makes a read, and a Workflow's append and close, on a Store with a Rust runtime + itself, with no Activity, so these options don't apply to them. Core tries such a + call again while its Store is unavailable, for at most 30 s. A :py:class:`StreamConfig` holds them in :py:attr:`StreamConfig.workflow_call_options`, and every Store of that config @@ -69,8 +108,15 @@ class StreamStore(ABC): :py:class:`temporalio.contrib.redis.streams.RedisStreamStore`, or :py:class:`temporalio.testing.MemoryStreamStore` in tests. - To keep Streams in another system, write a Store that implements :py:attr:`name` - and the five operations. Each operation takes and gives a message of + The SDK's Stores run on the bridge, in Rust, and this class isn't meant to be + subclassed in Python. A Store that is written in Python works for a client and an + Activity, for example as a double in a test. Workflow code loses the Direct calls on + it. An append and a close of Workflow code then run in a Local Activity, with one + marker in History for each call, and a read fails the Workflow, so Workflow code + can't read its Streams. + + Such a Store implements :py:attr:`name` and the five operations. Each operation + takes and gives a message of ``temporalio.api.streamservice.v1``. The bodies of the Records arrive as payloads that the SDK already encoded with the data converter and the payload codec, so a Store keeps payloads and not the user's types. A failed call raises @@ -157,6 +203,11 @@ async def call_from_workflow( task fails after a close, the Stream stays closed. The retry of the task then can't append to it. + On a Store with a Rust runtime, such as the Redis Store and + :py:class:`temporalio.testing.MemoryStreamStore`, Core appends and closes for + Workflow code itself, and this method gets no append and no close of a Producer + of kind Workflow. + Args: operation: The Nexus operation name, ``CreateStream``, ``AppendRecords``, ``CloseStream`` or ``DescribeStream`` in this release. @@ -183,8 +234,8 @@ async def call_from_workflow( class _BridgeStreamStore(StreamStore): # pyright: ignore[reportUnusedClass] """A Store that runs on the Stream runtime of the bridge. - It builds the runtime at its first call, so the Store can be built before the - Runtime and its Tokio threads exist. + It builds the runtime at its first call, or when a Worker with Workflows gives it to + Core, so the Store can be built before the Runtime and its Tokio threads exist. """ def __init__( @@ -237,6 +288,18 @@ async def describe_stream( """See base class.""" return await self._call_bridge("DescribeStream", request) + def _get_stream_runtime(self) -> temporalio.bridge.streams.StreamRuntime: + """Give the Stream runtime of the bridge, and build it at the first use. + + Raises: + temporalio.exceptions.StreamConfigError: This build of temporalio has no + Stream runtime, or not this Store. + """ + if self._stream_runtime is None: + runtime = self._runtime or temporalio.runtime.Runtime.default() + self._stream_runtime = self._new_stream_runtime(runtime._core_runtime) + return self._stream_runtime + async def _call_bridge( self, operation: str, request: google.protobuf.message.Message ) -> Any: @@ -247,11 +310,8 @@ async def _call_bridge( temporalio.exceptions.StreamConfigError: This build of temporalio has no Stream runtime, or not this Store. """ - if self._stream_runtime is None: - runtime = self._runtime or temporalio.runtime.Runtime.default() - self._stream_runtime = self._new_stream_runtime(runtime._core_runtime) # The bridge takes and gives the messages in their wire form. - response = await self._stream_runtime.call( + response = await self._get_stream_runtime().call( operation, request.SerializeToString() ) return _STREAM_OPERATIONS[operation].response_type.FromString(response) @@ -268,9 +328,6 @@ class _StreamOperation: response_type: type[google.protobuf.message.Message] - workflow_refusal: str | None = None - """Why Workflow code can't start it in this release, or ``None`` when it can.""" - async def call( self, store: StreamStore, request: google.protobuf.message.Message ) -> google.protobuf.message.Message: @@ -308,10 +365,6 @@ def check_response( "read_records", temporalio.api.streamservice.v1.ReadRecordsRequest, temporalio.api.streamservice.v1.ReadRecordsResponse, - workflow_refusal=( - "Workflow code can't read a Stream on a Store in this release. " - "Call an Activity that reads it through its client." - ), ), "CloseStream": _StreamOperation( "close_stream", @@ -437,10 +490,53 @@ class StreamConfig: Task. The appends and the close of one Workflow on one Stream run one at a time, in the order of the calls. - Workflow code calls a Store from a Local Activity, also for a close. So it needs - no Worker that polls for Activity Tasks. A Local Activity reaches the Store before - its Workflow Task completes. If that task fails after a close, the Stream stays - closed. The retry of the task then can't append to it. + On a Store with a Rust runtime, Core appends and closes for Workflow code itself, + with no Activity. Core closes the Stream only after the server took the Workflow Task + of the close, so a failed task closes nothing. On a Store written in Python, and for + a caller's Producer of another kind, Workflow code calls the Store from a Local + Activity, also for a close. So it needs no Worker that polls for Activity Tasks. A + Local Activity reaches the Store before its Workflow Task completes. If that task + fails after a close, the Stream stays closed. The retry of the task then can't append + to it. + + Workflow code reads with the same ``read``, ``iter_read`` and ``iter_read_items`` as + a client. On a Store with a Rust runtime, Core reads the Store inside the Workflow + Task. A page that Workflow code gets in one activation has at most 256 Records, and + the rest come in the next activation with no wait. A loop that asks for its next page in the same + activation keeps one read open in Core. A read that no code waits on closes before + the activation completes, so it never holds a Workflow Task open. + + .. code-block:: python + + async for record in events.iter_read_items(wait=timedelta(seconds=10)): + if record.body.kind == "tool_call": + tools.append( + workflow.start_activity( + run_tool, + record.body.tool_call, + start_to_close_timeout=timedelta(minutes=1), + ) + ) + + The loop ends after the page that is ``done``, when the Stream is closed and the + read gave its last Record. With a ``wait``, a quiet Stream gives empty pages, and + the loop goes on. So a loop on a Stream that can be quiet for long uses + ``iter_read`` and leaves after an empty page. A read has these costs. + + * A call keeps its Workflow Task open while it waits for its page. A Signal, an + Update, a cancel or an Activity result reaches the Workflow at the next Workflow + Task, at the heartbeat of the task at the latest. That is about 8 s with the + default task timeout of 10 s. + * The calls of one Workflow Task write one marker in History, whatever the number of + Records. + * With the default :py:attr:`StreamReplaySource.STORE`, History holds no Record. A + replay reads the Records from the Store again, so they must stay in the Store for + as long as the Workflow can replay. + * A replay on a Worker with no cache reads the Store again on every Workflow Task. + + On a Store written in Python, a read fails the Workflow with a non-retryable + :py:class:`temporalio.exceptions.ApplicationError`. Code in a Query handler or an + Update validator can't read. .. warning:: This API is experimental and subject to change. @@ -480,6 +576,31 @@ class StreamConfig: Local Activity with their timeouts and retry policy. """ + workflow_read_replay_from: StreamReplaySource = dataclasses.field( + default=StreamReplaySource.STORE + ) + """Where a replay of a read of Workflow code gets the Records of the read. + + It applies to every read of Workflow code on a Worker whose client has this config, + and decides what a new read records in History. :py:attr:`StreamReplaySource.STORE`, + the default, puts only the range of offsets, the count and a digest of the Records + into History, so a replay reads them from the Store again. The Records must then stay + in the Store for as long as the Workflow can replay. The option + :py:attr:`StreamReplaySource.HISTORY` puts the Records with their bodies into + History, so a replay needs no Store. A replay follows the marker in History and not + this setting, so a Workflow that is running replays after a change of it. + """ + + workflow_read_ahead_bytes: int | None = None + """The most bytes of one read's Records that wait in the Worker for Workflow code. + + A read of Workflow code stops asking the Store while this many bytes of its Records + wait in the Worker's memory for Workflow code to take them. The read goes + on when Workflow code takes Records. The first page of a read always comes. ``None`` + takes Core's default of 4 MiB. Set a larger value for large Records, and a smaller one + for a Worker with little memory, since each open read can hold this much. + """ + _store_map: dict[str, StreamStore] = dataclasses.field( init=False, repr=False, compare=False ) @@ -489,8 +610,14 @@ def __post_init__(self) -> None: """Check the Stores and build the name-keyed map of them. Raises :py:class:`temporalio.exceptions.StreamConfigError` if there is no - Store, or if two Stores share a name. + Store, or if two Stores share a name, and ``ValueError`` if + :py:attr:`workflow_read_ahead_bytes` is below 1. """ + if ( + self.workflow_read_ahead_bytes is not None + and self.workflow_read_ahead_bytes < 1 + ): + raise ValueError("workflow_read_ahead_bytes must be at least 1") if not self.stores: raise StreamConfigError("StreamConfig.stores must have at least one Store") store_map: dict[str, StreamStore] = {} @@ -532,6 +659,20 @@ def _get_store_by_name(self, name: str) -> StreamStore: ) return store + def _bridge_stream_runtimes( + self, + ) -> list[tuple[str, temporalio.bridge.streams.StreamRuntime]]: + """Give the Stream runtime of each Store that runs on the bridge, by Store name. + + Core reads and appends a Stream of Workflow code through these runtimes. A Store + that a user wrote in Python has none. + """ + return [ + (store.name, store._get_stream_runtime()) + for store in self.stores + if isinstance(store, _BridgeStreamStore) + ] + def _store_for( self, ref: temporalio.nexus.system.stream_service.StreamRef[Any], diff --git a/temporalio/contrib/redis/streams.py b/temporalio/contrib/redis/streams.py index 8a366fe5f..ab8136d84 100644 --- a/temporalio/contrib/redis/streams.py +++ b/temporalio/contrib/redis/streams.py @@ -31,7 +31,9 @@ class RedisStreamStore(temporalio.client._stream_store._BridgeStreamStore): It runs the Stream runtime of the bridge on the Redis Store of the ``temporalio-streams-redis`` crate. It connects at its first call, so building it needs no server, and a server that is down fails that call. A failed call - raises :py:class:`temporalio.exceptions.StreamError`. + raises :py:class:`temporalio.exceptions.StreamError`. A Worker with Workflows gives + Core this runtime when the Worker is built, so Core can read and append for Workflow + code. A key has no namespace, so a Stream id is unique in one Redis and key prefix. Give each namespace its own key prefix to keep their Streams apart. @@ -54,10 +56,12 @@ def __init__( name: str = "redis", runtime: temporalio.runtime.Runtime | None = None, ) -> None: - """Create a Redis Store. It builds its Stream runtime and connects at its first call. + """Create a Redis Store. It builds its Stream runtime at its first use. - A bad option raises ``ValueError`` at that first call, with the message of the - Redis crate. + The first use is the first call, or the build of a Worker with Workflows on a + client with this Store, which gives Core the runtime. A bad option raises + ``ValueError`` there, with the message of the Redis crate. The Store connects at + its first call. Args: url: A ``redis://`` or ``rediss://`` URL, or the seed URLs of a cluster. @@ -79,7 +83,7 @@ def __init__( name: Name of the Store, which the refs of its Streams carry, so that every process can route a ref that it gets. runtime: Runtime whose Tokio threads run the Store. ``None`` takes - :py:meth:`temporalio.runtime.Runtime.default` at the first call. + :py:meth:`temporalio.runtime.Runtime.default` at the first use. """ temporalio.client._stream_store._check_not_negative( retention=retention, diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 2e013f666..273cf57fb 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -42,3 +42,130 @@ async def write_parts(ref: StreamRef[str], parts: list[str]) -> None: The retry's Records continue the first attempt's epoch, so a reader does not take them as a replacement of the first attempt's Records. + +## Reads and appends in Workflow code + +The helpers run on the Streams API, so they follow how Workflow code reads, appends and +closes. On a Store with a Rust runtime, such as the Redis Store or `MemoryStreamStore`, Core +reads, appends and closes a Stream itself, with no Activity. Workflow code reads with the same +`read`, `iter_read` and `iter_read_items` as a client. It gets the same pages, but each page has +at most 256 Records. + +Here the Workflow reads the events of a model's answer while the model Activity still runs, +and starts each tool as its call arrives. + +```python +events = workflow.get_stream(events_ref, body_type=ModelEvent) +status = workflow.get_stream(status_ref, body_type=str) +tools = [] +async for record in events.iter_read_items(wait=timedelta(seconds=10)): + event = record.body + if event.kind == "tool_call": + # The tool starts now, before the model ends its answer. + tools.append( + workflow.start_activity( + run_tool, event.tool_call, start_to_close_timeout=timedelta(minutes=1) + ) + ) + await status.append(f"Running {event.tool_call.name}") +results = await asyncio.gather(*tools) +``` + +The loop ends after the page that is `done`, when the Stream is closed and the read gave its +last Record. With no `wait`, a call gives what is in the Stream, and the loop ends at the first +empty page. With a `wait`, a call on a quiet Stream gives an empty page after the wait, and +the loop asks again. So a loop on a Stream that can be quiet for long uses `iter_read` and +leaves after an empty page. A later loop from the `last_offset` of that page goes on there. + +```python +last = None +async for page in tokens.iter_read(from_offset=last, wait=timedelta(seconds=30)): + for record in page.records: + await shown.append(record.body.text) + last = page.last_offset + if not page.records: + # The Stream was quiet for 30 s, or it is done. + break +``` + +Core keeps one read open while a loop asks for its next page in the same activation. A loop +that awaits something else between two pages, as the `append` above does, lets the read close +before the activation completes, and its next page starts a new read. A read that no code +waits on never holds a Workflow Task open. + +The calls of one Workflow Task write one marker in History, whatever the number of Records. A +read has three other costs. + +- A call keeps its Workflow Task open while it waits for its page. A Signal, an Update, a + cancel or an Activity result reaches the Workflow at the next Workflow Task, at the + heartbeat of the task at the latest. That is about 8 s with the default task timeout of + 10 s. A call with a `wait` on a quiet Stream holds its task open for the whole wait. +- By default, `StreamReplaySource.STORE`, the marker holds a range of offsets, a count and a + digest, and no Record. A replay reads the Store again, so the Records must stay in the + Store for as long as the Workflow can replay. A replay whose Store stays unavailable for + 5 s fails only its Workflow Task, with `StreamReplayUnavailable`, and never the Workflow. + The option + `workflow_read_replay_from=StreamReplaySource.HISTORY` of `StreamConfig` puts the Records + into the marker, so a replay needs no Store. Then one Workflow Task's marker holds at most + 1 MiB of Records. Past that, the task completes and the read goes on in the next one. +- A replay on a Worker with no cache reads the Store again on every Workflow Task. + +While Workflow code is busy, a read holds at most 4 MiB of its Records in the Worker, then +waits until the code takes them. `workflow_read_ahead_bytes` of `StreamConfig` changes that +bound, for large Records or a Worker with little memory. A read gives Workflow code its Records +as soon as no activation of the Workflow is pending. A page that Workflow code gets in one +activation has at most 256 Records, and the rest come in the next activation with no wait. + +An append and a close of Workflow code always write as the Workflow, with the Workflow Task as +their epoch. An append runs inside its Workflow Task and records a marker with no body. An +append is at-least-once: when a Worker crashes or its run is evicted before the task with the +append completes, the rerun of that task writes a second copy with the epoch of the new task. + +Core tries a read, an append or a close again while its Store answers `StorageUnavailable`. +It waits from 100 ms up to 2 s between tries, for at most 30 s. Only then does Workflow code +get the retryable `StreamError`. Every other reason raises on the first try. An append of +Workflow code always carries a `sequence`, so the Store writes its Records once, also when a +try reached the Store but its answer got lost. + +A close spans two Workflow Tasks. The task that calls `close` records only the intent. Core +closes the Stream after the server took that task, and the next task gives the result. So a +failed task closes nothing, and its retry can still append. A Workflow that awaits its close +never completes in the task that asked for it. Await the close, also in a `finally` block. + +```python +stream = await workflow.create_stream(id=stream_id, body_type=str) +try: + await stream.append("a") +finally: + await stream.close() +``` + +A close that the code doesn't await still happens when the Workflow ends in the same task, +after the server took that task. A graceful shutdown of the Worker waits for such a close, for +at most 10 s. Only a Worker that dies there, or a Store that doesn't answer in that time, loses +the close, and the Stream stays open. + +A Store isn't meant to be written in Python, since such a Store has no Rust runtime. On one, +an append and a close of Workflow code run in the Store's Local Activity, and a read fails the +Workflow. + +## Metrics + +Core records these metrics for the Direct calls of Workflow code. Their prefix is the metric +prefix of the `Runtime`, which is `temporal_` by default. Each metric has the attribute +`stream_store` with the Store of the call, next to the `namespace`, `task_queue` and +`workflow_type` that a Local Activity metric has. A replay that takes a result from a marker +counts nothing. + +| Metric | What it counts | +| --- | --- | +| `temporal_stream_read_total` | Reads that Core started | +| `temporal_stream_read_failed` | Reads that ended with a failure | +| `temporal_stream_read_records` | Records that reads gave to Workflow code | +| `temporal_stream_append_total`, `_failed` | Appends that Core sent, and those that failed | +| `temporal_stream_append_latency` | From the start of an append in Core to its result | +| `temporal_stream_close_total`, `_failed` | Closes that Core sent, and those that failed | +| `temporal_stream_close_latency` | From the start of a close in Core to its result | +| `temporal_stream_close_abandoned` | Closes that a graceful shutdown gave up after 10 s | +| `temporal_stream_replay_read_total`, `_failed` | Marker entries that a replay read again | +| `temporal_stream_replay_read_latency` | The time of one such read and its check | diff --git a/temporalio/exceptions.py b/temporalio/exceptions.py index 658012ee4..450543b47 100644 --- a/temporalio/exceptions.py +++ b/temporalio/exceptions.py @@ -218,7 +218,10 @@ class StreamError(ApplicationError): ``STREAM_ERROR_REASON_CLOSED`` (the Stream is closed), or ``STREAM_ERROR_REASON_ALREADY_EXISTS`` (a strict create of a Stream that exists). A Store outage, ``STREAM_ERROR_REASON_STORAGE_UNAVAILABLE``, is the one retryable - reason. + reason. On a Store with a Rust runtime, Core tries a read, an append or a close of + Workflow code again while the Store is unavailable. It waits from 100 ms up to 2 s + between tries, and Workflow code gets this reason only after 30 s. Every other + reason raises on the first try. .. warning:: This API is experimental and subject to change. @@ -304,11 +307,37 @@ class StreamConfigError(RuntimeError): * This build of temporalio has no Stream runtime, or no Redis Stream Store. A Workflow's Stream call on a Worker whose client has no Stream config doesn't - raise this class. Workflow code gets an :py:class:`ApplicationError` with the - ``type`` ``StreamConfigError`` and ``non_retryable`` set, so an uncaught one fails - the Workflow. Workflow code can catch this error. A Workflow that catches it must - not go on to make the same call on a Worker that has a config, since the History - then has no record of the first call. + raise this class, and neither does a read of Workflow code on a Store written in + Python, which has no Rust runtime for Core to read through. Workflow code gets an + :py:class:`ApplicationError` with the ``type`` ``StreamConfigError`` and + ``non_retryable`` set, so an uncaught one fails the Workflow. Workflow code can + catch this error. A Workflow that catches it must not go on to make the same call on + a Worker that has a config, since the History then has no record of the first call. + + .. warning:: + This API is experimental and subject to change. + """ + + +class StreamReplayError(TemporalError): + """A replay could not read the Records of a read of Workflow code from its Store again. + + A read under the default :py:attr:`temporalio.client.StreamReplaySource.STORE` keeps + no Record in History, so a replay reads the Records from the Store again. When the + Store doesn't have them, for example after the Stream's Retention, or they differ from + what History says, the replay can't go on. + + By default that fails only the Workflow Task, so a later attempt works once the + Records are back. With this class in ``workflow_failure_exception_types`` of the + Worker, or in ``failure_exception_types`` of ``@workflow.defn``, the Workflow fails + instead. Its failure is an :py:class:`ApplicationError` with the ``type`` + ``StreamReplayError``, and Core raises no instance of this class. + + A Store that doesn't answer is not this error, since the Records may still be + there. A replay tries the Store again for 5 s. If it stays unavailable, only the + Workflow Task fails, with a message that starts with ``StreamReplayUnavailable``. + That holds also with this class in ``workflow_failure_exception_types`` or + ``failure_exception_types``, so a later attempt of the task can read the Records. .. warning:: This API is experimental and subject to change. diff --git a/temporalio/nexus/system/stream_service/workflow/_poll.py b/temporalio/nexus/system/stream_service/workflow/_poll.py new file mode 100644 index 000000000..a653d84f9 --- /dev/null +++ b/temporalio/nexus/system/stream_service/workflow/_poll.py @@ -0,0 +1,77 @@ +# Generated by nexgen v0.2.12. DO NOT EDIT! + +from __future__ import annotations + +import collections +import collections.abc +import typing + +_PollRequestT = typing.TypeVar("_PollRequestT") +_PollResponseT = typing.TypeVar("_PollResponseT") +_PollItemT = typing.TypeVar("_PollItemT") + + +@typing.final +class PollIterator(typing.Generic[_PollRequestT, _PollItemT]): + """Yields each item of the pages of a poll. + + ``next_request`` is the request to resume with. After an item, the resume helper gives + it. Without a resume helper, it reads the item's page again. After a page's last item, + it is the page helper's request, which is ``None`` after the last page. + """ + + def __init__( + self, + request: _PollRequestT, + pages: collections.abc.AsyncIterator[_PollResponseT], + items: collections.abc.Callable[ + [_PollResponseT], collections.abc.Sequence[_PollItemT] | None + ], + next_page: collections.abc.Callable[ + [_PollRequestT, _PollResponseT], _PollRequestT | None + ], + resume: collections.abc.Callable[[_PollRequestT, _PollItemT], _PollRequestT] + | None, + ) -> None: + async def read( + page_request: _PollRequestT, + ) -> tuple[_PollRequestT | None, collections.deque[_PollItemT]]: + response = await anext(pages) + # The page iterator keeps its requests to itself, so the page helper runs here + # again. It runs before the caller gets an item, so a changed item can't change + # the next request. + return next_page(page_request, response), collections.deque( + items(response) or () + ) + + self.next_request: _PollRequestT | None = request + self._read: collections.abc.Callable[ + [_PollRequestT], + collections.abc.Awaitable[ + tuple[_PollRequestT | None, collections.deque[_PollItemT]] + ], + ] = read + self._resume: ( + collections.abc.Callable[[_PollRequestT, _PollItemT], _PollRequestT] | None + ) = resume + self._page: collections.deque[_PollItemT] = collections.deque() + self._page_request: _PollRequestT = request + self._page_next: _PollRequestT | None = None + + def __aiter__(self) -> PollIterator[_PollRequestT, _PollItemT]: + return self + + async def __anext__(self) -> _PollItemT: + while not self._page: + if self.next_request is None: + raise StopAsyncIteration + self._page_request = self.next_request + self._page_next, self._page = await self._read(self._page_request) + if not self._page: + self.next_request = self._page_next + item = self._page.popleft() + if not self._page: + self.next_request = self._page_next + elif self._resume is not None: + self.next_request = self._resume(self._page_request, item) + return item diff --git a/temporalio/nexus/system/stream_service/workflow/_resources/stream.py b/temporalio/nexus/system/stream_service/workflow/_resources/stream.py index df548d252..0aa32cc38 100644 --- a/temporalio/nexus/system/stream_service/workflow/_resources/stream.py +++ b/temporalio/nexus/system/stream_service/workflow/_resources/stream.py @@ -2,6 +2,8 @@ from __future__ import annotations +import collections.abc +import datetime import types import typing @@ -20,8 +22,12 @@ DescribeStreamRequest, DescribeStreamResponse, Producer, + ReadRecordsRequest, + ReadRecordsResponse, + StreamRecord, StreamRef, ) +from .._poll import PollIterator BodyT = typing.TypeVar("BodyT") @@ -86,6 +92,103 @@ async def close( handle = await _close_stream(request) await handle + async def read( + self, + from_offset: bytes | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + ) -> ReadRecordsResponse[BodyT]: + """Read one page of Records. The poll iterator reads them all.""" + request = ReadRecordsRequest( + ref=self.ref, + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, + ) + handle = await _read_records(request, self.body_type) + return await handle + + async def iter_read( + self, + from_offset: bytes | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + ) -> collections.abc.AsyncGenerator[ReadRecordsResponse[BodyT], None]: + """Yield each response of ``read`` until the poll helper stops. + + Read one page of Records. The poll iterator reads them all. + + Args: + from_offset: Read after this offset. Empty is the beginning. A too high + offset fails. + max_records: About this many Records per page, 1 to 1000. + wait: How long to wait for a new Record before an empty answer. At most 60 + s. + max_bytes: The maximum Record bytes in a page. A larger Record is still + returned alone. + """ + request = ReadRecordsRequest( + ref=self.ref, + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, + ) + while True: + handle = await _read_records(request, self.body_type) + response = await handle + next_request = _support.next_read_request(request, response) + yield response + if next_request is None: + return + request = next_request + + def iter_read_items( + self, + from_offset: bytes | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + ) -> PollIterator[ReadRecordsRequest[BodyT], StreamRecord[BodyT]]: + """Yield each item of ``iter_read``, page by page. + + ``next_request`` resumes after the last item that the iterator yielded. It is + ``None`` after the last page. + + Read one page of Records. The poll iterator reads them all. + + Args: + from_offset: Read after this offset. Empty is the beginning. A too high + offset fails. + max_records: About this many Records per page, 1 to 1000. + wait: How long to wait for a new Record before an empty answer. At most 60 + s. + max_bytes: The maximum Record bytes in a page. A larger Record is still + returned alone. + """ + request = ReadRecordsRequest( + ref=self.ref, + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, + ) + return PollIterator( + request, + self.iter_read( + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, + ), + lambda response: response.records, + _support.next_read_request, + _support.resume_read_request, + ) + async def describe( self, ) -> DescribeStreamResponse: @@ -164,6 +267,21 @@ async def _close_stream( ) +async def _read_records( + request: ReadRecordsRequest[BodyT], + body_type: type[BodyT], +) -> temporalio.workflow.NexusOperationHandle[ReadRecordsResponse[BodyT],]: + nexus_client = temporalio.workflow.create_nexus_client( + service="temporal.api.streamservice.v1.StreamService", + endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, + ) + return await nexus_client.start_operation( + operation="ReadRecords", + input=request, + output_type=ReadRecordsResponse[body_type], + ) + + async def _describe_stream( request: DescribeStreamRequest[BodyT], ) -> temporalio.workflow.NexusOperationHandle[DescribeStreamResponse,]: diff --git a/temporalio/testing/_stream_store.py b/temporalio/testing/_stream_store.py index 1220bbc5b..42c8500fd 100644 --- a/temporalio/testing/_stream_store.py +++ b/temporalio/testing/_stream_store.py @@ -28,7 +28,7 @@ def __init__( producer_idle_ttl: timedelta | None = None, runtime: temporalio.runtime.Runtime | None = None, ) -> None: - """Create a memory Store. It builds its Stream runtime at its first call. + """Create a memory Store. It builds its Stream runtime at its first use. Args: name: Name of the Store, which the refs of its Streams carry. @@ -37,7 +37,7 @@ def __init__( producer_idle_ttl: How long an open Stream stays after the last write of any Producer. ``None`` keeps an open Stream until its close. runtime: Runtime whose Tokio threads run the Store. ``None`` takes - :py:meth:`temporalio.runtime.Runtime.default` at the first call. + :py:meth:`temporalio.runtime.Runtime.default` at the first use. """ temporalio.client._stream_store._check_not_negative( retention=retention, producer_idle_ttl=producer_idle_ttl diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index 888c06f29..058d80cd7 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -16,8 +16,13 @@ ResolveNexusOperationStart, ResolveRequestCancelExternalWorkflow, ResolveSignalExternalWorkflow, + ResolveStreamAppend, + ResolveStreamClose, + ResolveStreamRead, + StreamReadEnd, ) from temporalio.bridge.proto.workflow_commands.workflow_commands_pb2 import ( + AppendToStream, CompleteWorkflowExecution, FailWorkflowExecution, ScheduleActivity, @@ -37,6 +42,8 @@ class CommandInfo: command_type: CommandType.ValueType command_seq: int update_protocol_instance_id: str | None = None + stream: bool = False + """Whether the payloads are the bodies of a Stream append or the Records of a read.""" current_command_info: contextvars.ContextVar[CommandInfo | None] = ( @@ -57,6 +64,7 @@ def __init__( skip_search_attributes: bool = False, skip_headers: bool = False, concurrency_limit: int = 1, + skip_stream_bodies: bool = False, ) -> None: """Creates a new command-aware payload visitor. @@ -65,12 +73,53 @@ def __init__( skip_headers: If True, headers are not visited. concurrency_limit: Maximum number of payload visits that may run concurrently during a single call to visit(). Defaults to 1. + skip_stream_bodies: If True, the bodies of a Stream append are not + visited. """ super().__init__( skip_search_attributes=skip_search_attributes, skip_headers=skip_headers, concurrency_limit=concurrency_limit, ) + self.skip_stream_bodies = skip_stream_bodies + + # A Store keeps the bodies of a Workflow's append as the Local Activity path, a client + # and an Activity write them: with the codec, with no serialization context and with no + # external storage. So every reader decodes them. Each body is one Record, so the codec + # gets each body alone, as for those paths. + async def _visit_coresdk_workflow_commands_AppendToStream( + self, fs: VisitorFunctions, o: AppendToStream + ) -> None: + if self.skip_stream_bodies: + return + with current_command(CommandType.COMMAND_TYPE_UNSPECIFIED, o.seq, stream=True): + for body in o.bodies: + await self._visit_temporal_api_common_v1_Payload(fs, body) + + # The Records of a read come from any writer of the Stream, so the codec decodes them + # with no serialization context, as a client and an Activity do. + async def _visit_coresdk_workflow_activation_ResolveStreamRead( + self, fs: VisitorFunctions, o: ResolveStreamRead + ) -> None: + with current_command(CommandType.COMMAND_TYPE_UNSPECIFIED, o.seq, stream=True): + await super()._visit_coresdk_workflow_activation_ResolveStreamRead(fs, o) + + # Core builds the failure of a Stream read, append or close itself, with no codec, so no + # pass may decode it. + async def _visit_coresdk_workflow_activation_StreamReadEnd( + self, fs: VisitorFunctions, o: StreamReadEnd + ) -> None: + pass + + async def _visit_coresdk_workflow_activation_ResolveStreamAppend( + self, fs: VisitorFunctions, o: ResolveStreamAppend + ) -> None: + pass + + async def _visit_coresdk_workflow_activation_ResolveStreamClose( + self, fs: VisitorFunctions, o: ResolveStreamClose + ) -> None: + pass # Workflow commands with payloads async def _visit_coresdk_workflow_commands_CompleteWorkflowExecution( @@ -218,6 +267,7 @@ def current_command( command_seq: int, *, update_protocol_instance_id: str | None = None, + stream: bool = False, ) -> Iterator[None]: """Context manager for setting command info.""" token = current_command_info.set( @@ -225,6 +275,7 @@ def current_command( command_type=command_type, command_seq=command_seq, update_protocol_instance_id=update_protocol_instance_id, + stream=stream, ) ) try: diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index 7ad6bdeaa..513fe2abb 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -22,7 +22,11 @@ from ..common import HeaderCodecBehavior from ._interceptor import Interceptor -from ._stream_store import _WorkflowStreamStore +from ._stream_store import ( + _bridge_read_ahead_bytes, + _bridge_stream_runtimes, + _WorkflowStreamStore, +) from ._worker import load_default_build_id from ._workflow import _WorkflowWorker from ._workflow_instance import ( @@ -68,9 +72,19 @@ def __init__( ``stream_config`` takes the Stream config of the client that the worker had, so a replay makes the same commands. A history that records a - Stream call needs it. With no Stream config, Workflow code gets an error in - place of that call, and the replay fails with a nondeterminism error. A - replay calls no method of its Stores. + create, a describe, or an append or a close through a Store's Local Activity + needs it. With no Stream config, Workflow code gets an error in place of that + call, and the replay fails with a nondeterminism error. A replay calls no method + of its Stores from Python. An append or a close that Core made needs no Stream + config, and a replay closes nothing. A read of Workflow code that ran under the + option ``HISTORY`` of ``workflow_read_replay_from`` needs none either. That holds + also when the ref has no Store, since Core takes the Store from the history. A + read that ran under the default, ``StreamReplaySource.STORE``, has no Records in + the history. Core reads them from the Store again, so the replay needs this + config and Stores that still have the Records. Without them the replay fails + with a :py:class:`RuntimeError` whose message has + ``StreamReplayError``, the Stream and the cause. A Store that stays unavailable + for 5 s gives ``StreamReplayUnavailable`` in place of ``StreamReplayError``. Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that @@ -319,6 +333,10 @@ def on_eviction_hook( # per workflow type nondeterminism_as_workflow_fail=workflow_worker.nondeterminism_as_workflow_fail(), nondeterminism_as_workflow_fail_for_types=workflow_worker.nondeterminism_as_workflow_fail_for_types(), + stream_replay_as_workflow_fail=workflow_worker.stream_replay_as_workflow_fail(), + stream_replay_as_workflow_fail_for_types=( + workflow_worker.stream_replay_as_workflow_fail_for_types() + ), # All values below are ignored but required by Core max_cached_workflows=2, tuner=temporalio.bridge.worker.TunerHolder( @@ -368,6 +386,12 @@ def on_eviction_hook( (plugin.name(), plugin.version()) for plugin in self.plugins ], storage_drivers=storage_driver_types, + stream_runtimes=_bridge_stream_runtimes( + self._config.get("stream_config") + ), + stream_read_ahead_bytes=_bridge_read_ahead_bytes( + self._config.get("stream_config") + ), ), ) bridge_worker_scope = bridge_worker diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py index daaac73e6..13881eb9b 100644 --- a/temporalio/worker/_stream_store.py +++ b/temporalio/worker/_stream_store.py @@ -3,17 +3,28 @@ from __future__ import annotations import asyncio +import collections import dataclasses -from collections.abc import Generator +from collections.abc import Generator, Iterable +from datetime import timedelta from typing import TYPE_CHECKING, Any +import google.protobuf.message from nexusrpc import OutputT +import temporalio.api.enums.v1 +import temporalio.api.failure.v1 +import temporalio.api.stream.v1 +import temporalio.api.streamservice.v1 +import temporalio.bridge.proto.workflow_commands import temporalio.exceptions import temporalio.workflow from temporalio.client._stream_store import ( _STREAM_OPERATIONS, StreamConfig, + StreamReplaySource, + StreamStore, + _BridgeStreamStore, _request_ref, ) from temporalio.converter import StorageDriverWorkflowInfo @@ -41,6 +52,333 @@ def _no_stream_config_error() -> temporalio.exceptions.ApplicationError: # pyri _ORDERED_OPERATIONS = frozenset({"AppendRecords", "CloseStream"}) +def _no_direct_read_error(store: StreamStore) -> temporalio.exceptions.ApplicationError: + # Core reads the Store inside the Workflow Task, and it can't call a Store that runs + # in Python. Each retry of the task would fail the same way for good, so the Workflow + # fails instead, as for a Worker with no Stream config. + return temporalio.exceptions.ApplicationError( + f"The Stream Store '{store.name}' has no Direct read, since it is written in " + "Python. Read the Stream in an Activity, or use a Store that runs on the bridge, " + "such as the Redis Store or MemoryStreamStore.", + type="StreamConfigError", + non_retryable=True, + ) + + +_STREAM_ERROR_DETAILS_TYPE = ( + temporalio.api.stream.v1.StreamErrorDetails.DESCRIPTOR.full_name +) + + +def _stream_error_from_core( # pyright: ignore[reportUnusedFunction] + failure: temporalio.api.failure.v1.Failure, +) -> temporalio.exceptions.StreamError | None: + """Give the Store error of a failure that Core made, or ``None`` for another one.""" + # Core gives a Store error the token of its reason as its type, and puts the reason in + # its details with no codec. The Local Activity path gives the Store's message alone, + # so the token comes off the message. + info = failure.application_failure_info + for payload in info.details.payloads: + if payload.metadata.get("messageType") != _STREAM_ERROR_DETAILS_TYPE.encode(): + continue + details = temporalio.api.stream.v1.StreamErrorDetails.FromString(payload.data) + message = failure.message.removeprefix(f"{info.type}: ") + return temporalio.exceptions._stream_error_from_reason(details.reason, message) + return None + + +def _start_direct_append( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: temporalio.api.streamservice.v1.AppendRecordsRequest, + store_name: str, +) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Send an append of a Workflow Producer to Core, which makes it in the Workflow Task.""" + payload_converter, _ = instance._system_nexus_converters(input) + append = temporalio.bridge.proto.workflow_commands.AppendToStream( + stream=temporalio.api.stream.v1.StreamRef(id=request.ref.id, store=store_name), + bodies=[record.body for record in request.records], + producer=request.producer, + sequence=request.sequence, + ) + append.producer.workflow.ClearField("epoch") + appended = instance._start_stream_append(append) + + async def result() -> OutputT: + response = await appended + return payload_converter._from_transfer_value( + temporalio.api.streamservice.v1.AppendRecordsResponse( + first_offset=response.first_offset, + last_offset=response.last_offset, + ), + input.output_type, + ) + + return _StreamStoreOperationHandle( + instance.create_task(result(), name=f"stream store: {input.operation_name}") + ) + + +def _start_direct_close( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: temporalio.api.streamservice.v1.CloseStreamRequest, + store_name: str, +) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Send a close of a Workflow Producer to Core. + + Core records the intent in this Workflow Task, closes the Stream after the server took + the task, and gives the result in a later task. So a failed task closes nothing. + """ + payload_converter, _ = instance._system_nexus_converters(input) + close = temporalio.bridge.proto.workflow_commands.CloseStream( + stream=temporalio.api.stream.v1.StreamRef(id=request.ref.id, store=store_name), + producer=request.producer, + ) + close.producer.workflow.ClearField("epoch") + closed = instance._start_stream_close(close) + + async def result() -> OutputT: + await closed + return payload_converter._from_transfer_value( + temporalio.api.streamservice.v1.CloseStreamResponse(), input.output_type + ) + + return _StreamStoreOperationHandle( + instance.create_task(result(), name=f"stream store: {input.operation_name}") + ) + + +def _start_direct_call( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: google.protobuf.message.Message, + store_name: str, +) -> temporalio.workflow.NexusOperationHandle[OutputT] | None: + """Send an append or a close of a Workflow Producer to Core, or give ``None``. + + Core makes these calls itself, with the Workflow Task as the epoch of the Producer. + A Producer of another kind, and every other operation, keep the Activity path. + """ + if ( + isinstance(request, temporalio.api.streamservice.v1.AppendRecordsRequest) + and request.producer.WhichOneof("kind") == "workflow" + ): + return _start_direct_append(instance, input, request, store_name) + if ( + isinstance(request, temporalio.api.streamservice.v1.CloseStreamRequest) + and request.producer.WhichOneof("kind") == "workflow" + ): + return _start_direct_close(instance, input, request, store_name) + return None + + +def _start_replayed_direct_call( # pyright: ignore[reportUnusedFunction] + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], +) -> temporalio.workflow.NexusOperationHandle[OutputT] | None: + """Replay a Direct call on a Worker with no Stream config, or give ``None``. + + The marker of an append or a close holds its result, and the marker of a read can hold + its Records, so the replay needs no Store. A ref with no Store sends an empty name, and + Core takes the Store of History. + """ + if input.operation_name not in ("AppendRecords", "CloseStream", "ReadRecords"): + return None + payload_converter, _ = instance._system_nexus_converters(input) + request = payload_converter._to_transfer_value(input.input) + if isinstance(request, temporalio.api.streamservice.v1.ReadRecordsRequest): + # A replay of a read whose marker holds its Records reads no Store. + return _start_direct_read( + instance, input, request, request.ref.store, records_in_history=True + ) + return _start_direct_call(instance, input, request, request.ref.store) + + +# A client's read gives at most this many Records in a page, and a `max_records` of zero +# or less gives this many. A read of Workflow code takes the same limits. +_MAX_READ_RECORDS = 1000 +# The longest `wait` of a client's read. A longer one waits this long. +_MAX_READ_WAIT = timedelta(seconds=60) + + +@dataclasses.dataclass(frozen=True) +class _ReadKey: + """What a ``read`` call of Workflow code must match to take an open read.""" + + store: str + stream_id: str + offset: bytes + """The offset that the next page starts after. Empty is the start of the Stream.""" + wait: timedelta | None + + +class _DirectRead: # pyright: ignore[reportUnusedClass] + """One Core read of a Stream, which serves the ``read`` calls of Workflow code. + + Core gives its Records in batches. Each ``read`` call takes one page of them, and the + read then waits for the next call under the offset of that page. + """ + + def __init__(self, seq: int) -> None: + self.seq = seq + self.records: collections.deque[temporalio.api.stream.v1.StreamRecord] = ( + collections.deque() + ) + # Whether Core still reads, with no end and no close. + self.running = True + self.done = False + self.error: BaseException | None = None + # A job and an end wake the call that waits, which is at most one. + self._waiters: list[asyncio.Future[None]] = [] + + def kept(self) -> bool: + """Whether the read can serve a later call.""" + # A failure that no call raised yet stays, so the next call raises it. + return self.running or bool(self.records) or self.error is not None + + def take( + self, + records: Iterable[temporalio.api.stream.v1.StreamRecord], + *, + end: bool, + done: bool, + error: BaseException | None, + ) -> None: + """Keep the Records and the end of one job of Core.""" + self.records.extend(records) + if end: + self.running = False + self.done = done + self.error = error + self._wake() + + def close(self) -> None: + """Mark the read as closed in Core. Its buffered Records stay.""" + self.running = False + self._wake() + + def _wake(self) -> None: + for waiter in self._waiters: + if not waiter.done(): + waiter.set_result(None) + + async def page( + self, max_records: int, max_bytes: int | None + ) -> tuple[list[temporalio.api.stream.v1.StreamRecord], bool]: + """Give the next page and whether the Stream is done, and wait when none is buffered. + + Raises: + temporalio.exceptions.StreamError: The read failed. + """ + while not self.records and self.running: + waiter: asyncio.Future[None] = asyncio.get_running_loop().create_future() + self._waiters.append(waiter) + try: + await waiter + finally: + self._waiters.remove(waiter) + if not self.records and self.error is not None: + error, self.error = self.error, None + raise error + page: list[temporalio.api.stream.v1.StreamRecord] = [] + size = 0 + while self.records and len(page) < max_records: + record_size = self.records[0].ByteSize() + if page and max_bytes is not None and size + record_size > max_bytes: + break + page.append(self.records.popleft()) + size += record_size + return page, self.done and not self.records + + +def _read_limits( + request: temporalio.api.streamservice.v1.ReadRecordsRequest, +) -> tuple[int, timedelta | None, int | None]: + """Give the ``max_records``, ``wait`` and ``max_bytes`` of a client's read. + + Raises: + temporalio.exceptions.StreamError: ``max_bytes`` is negative. + """ + max_records = request.max_records + if max_records <= 0 or max_records > _MAX_READ_RECORDS: + max_records = _MAX_READ_RECORDS + wait: timedelta | None = None + if request.HasField("wait") and request.wait.ToTimedelta() > timedelta(): + wait = min(request.wait.ToTimedelta(), _MAX_READ_WAIT) + if request.max_bytes < 0: + raise temporalio.exceptions.StreamError( + "max_bytes must be 0 or more", + reason=temporalio.api.enums.v1.StreamErrorReason.STREAM_ERROR_REASON_INVALID, + ) + return max_records, wait, request.max_bytes or None + + +def _start_direct_read( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: temporalio.api.streamservice.v1.ReadRecordsRequest, + store_name: str, + *, + records_in_history: bool, +) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Answer one ``read`` call of Workflow code from a Core read of the Stream.""" + instance._assert_not_read_only("start stream read") + payload_converter, _ = instance._system_nexus_converters(input) + max_records, wait, max_bytes = _read_limits(request) + key = _ReadKey( + store=store_name, + stream_id=request.ref.id, + offset=request.from_offset, + wait=wait, + ) + read = instance._take_stream_read(key) + if read is None: + start = temporalio.bridge.proto.workflow_commands.StartStreamRead( + stream=temporalio.api.stream.v1.StreamRef( + id=request.ref.id, store=store_name + ), + from_offset=request.from_offset, + records_in_history=records_in_history, + ) + # The pages come from the calls, so the Core read has no cap and no timeout. + # With no `wait`, an idle timeout of zero gives what is in the Stream. + start.idle_timeout.SetInParent() + start.idle_timeout.FromTimedelta(wait or timedelta()) + read = instance._start_stream_read(start) + + async def result() -> OutputT: + try: + records, done = await read.page(max_records, max_bytes) + except BaseException: + instance._park_stream_read(read, key) + raise + last_offset = records[-1].offset if records else request.from_offset + instance._park_stream_read(read, dataclasses.replace(key, offset=last_offset)) + return payload_converter._from_transfer_value( + temporalio.api.streamservice.v1.ReadRecordsResponse( + records=records, last_offset=last_offset, done=done + ), + input.output_type, + ) + + return _StreamStoreOperationHandle( + instance.create_task(result(), name=f"stream store: {input.operation_name}") + ) + + +def _bridge_read_ahead_bytes(config: StreamConfig | None) -> int | None: # pyright: ignore[reportUnusedFunction] + """Give Core the read-ahead bound of the config, or ``None`` for Core's default.""" + return config.workflow_read_ahead_bytes if config is not None else None + + +def _bridge_stream_runtimes(config: StreamConfig | None) -> list[tuple[str, Any]]: # pyright: ignore[reportUnusedFunction] + """Give Core the bridge's object of each Store's Stream runtime, by Store name.""" + if config is None: + return [] + return [(name, runtime._ref) for name, runtime in config._bridge_stream_runtimes()] + + def _workflow_target() -> StorageDriverWorkflowInfo: # External storage builds the same value for a Workflow. info = temporalio.workflow.info() @@ -71,12 +409,6 @@ def start_operation( ) -> temporalio.workflow.NexusOperationHandle[OutputT]: """Answer a Stream call of Workflow code with the Store that the call goes to.""" stream_operation = _STREAM_OPERATIONS[input.operation_name] - # Workflow code can start any operation of the generated service through a Nexus - # client, also one that the Workflow `Stream` leaves out. - if stream_operation.workflow_refusal is not None: - raise temporalio.exceptions.ApplicationError( - stream_operation.workflow_refusal, non_retryable=True - ) payload_converter, _ = instance._system_nexus_converters(input) request = payload_converter._to_transfer_value(input.input) ref = _request_ref(input.input) @@ -84,6 +416,27 @@ def start_operation( # have raises `StreamConfigError` here, with no I/O, and fails the Workflow Task. store = self.config._store_for(ref, _workflow_target()) options = self.config.workflow_call_options + if isinstance(request, temporalio.api.streamservice.v1.ReadRecordsRequest): + instance._assert_not_read_only("start stream read") + if not isinstance(store, _BridgeStreamStore): + raise _no_direct_read_error(store) + return _start_direct_read( + instance, + input, + request, + store.name, + records_in_history=( + self.config.workflow_read_replay_from is StreamReplaySource.HISTORY + ), + ) + if isinstance(store, _BridgeStreamStore): + # Core appends and closes for a Workflow Producer itself, with the Workflow + # Task as its epoch. A Producer of another kind keeps the Local Activity path. + # Core orders the calls of one Stream, so a Direct call skips the order below. + direct = _start_direct_call(instance, input, request, store.name) + if direct is not None: + return direct + tails = instance._stream_write_tails tail_key = (store.name, ref.id) previous = ( diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index bbf68e2bf..a05ed0586 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -34,7 +34,11 @@ from ._interceptor import Interceptor from ._nexus import _NexusWorker from ._plugin import Plugin -from ._stream_store import _WorkflowStreamStore +from ._stream_store import ( + _bridge_read_ahead_bytes, + _bridge_stream_runtimes, + _WorkflowStreamStore, +) from ._tuning import WorkerTuner from ._workflow import ( _DEFAULT_WORKFLOW_TASK_EXTERNAL_STORAGE_CONCURRENCY, @@ -709,6 +713,13 @@ def check_activity(activity: str): if self._workflow_worker else set() ), + stream_replay_as_workflow_fail=self._workflow_worker is not None + and self._workflow_worker.stream_replay_as_workflow_fail(), + stream_replay_as_workflow_fail_for_types=( + self._workflow_worker.stream_replay_as_workflow_fail_for_types() + if self._workflow_worker + else set() + ), versioning_strategy=versioning_strategy, workflow_task_poller_behavior=_poller_behavior_to_bridge( workflow_task_poller_behavior @@ -724,6 +735,12 @@ def check_activity(activity: str): disable_payload_error_limit=config.get( "disable_payload_error_limit", False ), + # Only Workflow code makes Direct calls, so a Worker with no Workflows + # builds no runtime, as it builds no Store Activity. + stream_runtimes=_bridge_stream_runtimes( + stream_config if self._workflow_worker is not None else None + ), + stream_read_ahead_bytes=_bridge_read_ahead_bytes(stream_config), ), ) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 3e20e6adb..74e40dfd1 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -835,6 +835,12 @@ def nondeterminism_as_workflow_fail(self) -> bool: for typ in self._workflow_failure_exception_types ) + def stream_replay_as_workflow_fail(self) -> bool: + return any( + issubclass(temporalio.exceptions.StreamReplayError, typ) + for typ in self._workflow_failure_exception_types + ) + def _set_default_workflow_logic_flag( self, flag: _WorkflowLogicFlag, *, enabled: bool ) -> None: @@ -853,6 +859,16 @@ def nondeterminism_as_workflow_fail_for_types(self) -> set[str]: ) } + def stream_replay_as_workflow_fail_for_types(self) -> set[str]: + return { + k + for k, v in self._workflows.items() + if any( + issubclass(temporalio.exceptions.StreamReplayError, typ) + for typ in v.failure_exception_types + ) + } + class _DeadlockError(Exception): """Exception class for deadlocks. Contains functionality to swap the default traceback for another.""" diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 454a7c2c0..521a5d00c 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -95,7 +95,14 @@ WorkflowInboundInterceptor, WorkflowOutboundInterceptor, ) -from ._stream_store import _no_stream_config_error, _WorkflowStreamStore +from ._stream_store import ( + _DirectRead, + _no_stream_config_error, + _ReadKey, + _start_replayed_direct_call, + _stream_error_from_core, + _WorkflowStreamStore, +) logger = logging.getLogger(__name__) @@ -266,6 +273,17 @@ def get_thread_id(self) -> int | None: return None +def _is_completion( + command: temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.WorkflowCommand, +) -> bool: + return ( + command.HasField("complete_workflow_execution") + or command.HasField("continue_as_new_workflow_execution") + or command.HasField("fail_workflow_execution") + or command.HasField("cancel_workflow_execution") + ) + + class UnsandboxedWorkflowRunner(WorkflowRunner): """Workflow runner that does not do any sandboxing.""" @@ -364,6 +382,19 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._pending_nexus_operations: dict[int, _NexusOperationHandle] = {} self._pending_external_signals: dict[int, tuple[asyncio.Future, str]] = {} self._pending_external_cancels: dict[int, tuple[asyncio.Future, str]] = {} + # Core reads that run, by seq. + self._pending_stream_reads: dict[int, _DirectRead] = {} + # Reads that no call waits on, by what the next call must match. Each holds the + # Core read and the Records that it gave. + self._parked_stream_reads: dict[_ReadKey, list[_DirectRead]] = {} + self._pending_stream_appends: dict[ + int, + asyncio.Future[temporalio.bridge.proto.workflow_activation.StreamAppended], + ] = {} + self._pending_stream_closes: dict[int, asyncio.Future[None]] = {} + # Reads that the SDK closed. A job of one of them can still come in the activation + # that the close answers, and is dropped. + self._closed_stream_reads: set[int] = set() # Keyed by type self._curr_seqs: dict[str, int] = {} # TODO(cretz): Any concerns about not sharing this? Maybe the types I @@ -577,6 +608,17 @@ def activate( self._run_once( check_conditions=index == 1 or index == 2, jobs=act.jobs ) + + # A query-only activation sends no command, and the end of the Workflow + # stops every read in Core. + if ( + not job_sets[3] + and not self._deleting + and not any( + map(_is_completion, self._current_completion.successful.commands) + ) + ): + self._close_parked_stream_reads() except Exception as err: # We want some errors during activation, like those that can happen # during payload conversion, to be able to fail the workflow not the @@ -641,17 +683,7 @@ def activate( ) self._current_completion.failed.failure.application_failure_info.SetInParent() - def is_completion( - command: temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.WorkflowCommand, - ): - return ( - command.HasField("complete_workflow_execution") - or command.HasField("continue_as_new_workflow_execution") - or command.HasField("fail_workflow_execution") - or command.HasField("cancel_workflow_execution") - ) - - if any(map(is_completion, self._current_completion.successful.commands)): + if any(map(_is_completion, self._current_completion.successful.commands)): self._warn_if_unfinished_handlers() return self._current_completion @@ -693,6 +725,12 @@ def _apply( self._apply_resolve_signal_external_workflow( job.resolve_signal_external_workflow ) + elif job.HasField("resolve_stream_read"): + self._apply_resolve_stream_read(job.resolve_stream_read) + elif job.HasField("resolve_stream_append"): + self._apply_resolve_stream_append(job.resolve_stream_append) + elif job.HasField("resolve_stream_close"): + self._apply_resolve_stream_close(job.resolve_stream_close) elif job.HasField("signal_workflow"): self._apply_signal_workflow(job.signal_workflow) elif job.HasField("initialize_workflow"): @@ -944,6 +982,64 @@ def _apply_remove_from_cache( for task in self._tasks: task.cancel() + def _apply_resolve_stream_read( + self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamRead + ) -> None: + read = self._pending_stream_reads.get(job.seq) + if read is None: + if job.seq in self._closed_stream_reads: + return + raise RuntimeError(f"Failed finding Stream read for sequence {job.seq}") + error: BaseException | None = None + end = job.HasField("end") + if end: + del self._pending_stream_reads[job.seq] + if job.end.HasField("failure"): + error = _stream_error_from_core(job.end.failure) + if error is None: + error = self._context_free_failure_converter.from_failure( + job.end.failure, self._context_free_payload_converter + ) + read.take(job.records, end=end, done=job.end.done, error=error) + + def _apply_resolve_stream_append( + self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamAppend + ) -> None: + fut = self._pending_stream_appends.pop(job.seq, None) + if fut is None: + raise RuntimeError(f"Failed finding Stream append for sequence {job.seq}") + # Code that stopped waiting for the append has cancelled the future. + if fut.done(): + return + if job.HasField("appended"): + fut.set_result(job.appended) + return + error: BaseException | None = _stream_error_from_core(job.failed) + if error is None: + error = self._context_free_failure_converter.from_failure( + job.failed, self._context_free_payload_converter + ) + fut.set_exception(error) + + def _apply_resolve_stream_close( + self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamClose + ) -> None: + fut = self._pending_stream_closes.pop(job.seq, None) + if fut is None: + raise RuntimeError(f"Failed finding Stream close for sequence {job.seq}") + # Code that stopped waiting for the close has cancelled the future. + if fut.done(): + return + if not job.HasField("failed"): + fut.set_result(None) + return + error: BaseException | None = _stream_error_from_core(job.failed) + if error is None: + error = self._context_free_failure_converter.from_failure( + job.failed, self._context_free_payload_converter + ) + fut.set_exception(error) + def _apply_resolve_activity( self, job: temporalio.bridge.proto.workflow_activation.ResolveActivity ) -> None: @@ -1776,6 +1872,66 @@ async def workflow_start_child_workflow( ) ) + def _start_stream_read( + self, start: temporalio.bridge.proto.workflow_commands.StartStreamRead + ) -> _DirectRead: + start.seq = self._next_seq("stream_read") + self._add_command().start_stream_read.CopyFrom(start) + read = _DirectRead(start.seq) + self._pending_stream_reads[start.seq] = read + return read + + def _take_stream_read(self, key: _ReadKey) -> _DirectRead | None: + reads = self._parked_stream_reads.get(key) + if not reads: + return None + read = reads.pop(0) + if not reads: + del self._parked_stream_reads[key] + return read + + def _park_stream_read(self, read: _DirectRead, key: _ReadKey) -> None: + if read.kept(): + self._parked_stream_reads.setdefault(key, []).append(read) + + def _close_parked_stream_reads(self) -> None: + # A Core read holds its Workflow Task open, so one that no call waits on closes + # before the activation completes. Its Records stay for a later call. + for key, reads in list(self._parked_stream_reads.items()): + for read in reads: + if read.running: + self._add_command().close_stream_read.seq = read.seq + self._closed_stream_reads.add(read.seq) + del self._pending_stream_reads[read.seq] + read.close() + kept = [read for read in reads if read.kept()] + if kept: + self._parked_stream_reads[key] = kept + else: + del self._parked_stream_reads[key] + + def _start_stream_append( + self, append: temporalio.bridge.proto.workflow_commands.AppendToStream + ) -> asyncio.Future[temporalio.bridge.proto.workflow_activation.StreamAppended]: + command = self._add_command() + append.seq = self._next_seq("stream_append") + command.append_to_stream.CopyFrom(append) + fut: asyncio.Future[ + temporalio.bridge.proto.workflow_activation.StreamAppended + ] = self.create_future() + self._pending_stream_appends[append.seq] = fut + return fut + + def _start_stream_close( + self, close: temporalio.bridge.proto.workflow_commands.CloseStream + ) -> asyncio.Future[None]: + command = self._add_command() + close.seq = self._next_seq("stream_close") + command.close_stream.CopyFrom(close) + fut: asyncio.Future[None] = self.create_future() + self._pending_stream_closes[close.seq] = fut + return fut + def workflow_start_local_activity( self, activity: Any, @@ -2362,7 +2518,14 @@ async def _schedule_system_nexus_operation( if input.service == _STREAM_SERVICE.name: store = self._stream_store if store is None: - raise _no_stream_config_error() + replayed = ( + _start_replayed_direct_call(self, input) + if self._is_replaying + else None + ) + if replayed is None: + raise _no_stream_config_error() + return replayed return store.start_operation(self, input) return await self._outbound_start_nexus_operation(input) @@ -2558,6 +2721,11 @@ def get_serialization_context( workflow_id=self._info.workflow_id, ) + if command_info.stream: + # Stream bodies take the codec with no context on every path, so a client, an + # Activity and a Workflow read what each other wrote. + return None + if self._nexus_serialization_context and command_info.command_type in ( temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION, diff --git a/tests/contrib/redis/test_streams.py b/tests/contrib/redis/test_streams.py index b60a7e4df..742911199 100644 --- a/tests/contrib/redis/test_streams.py +++ b/tests/contrib/redis/test_streams.py @@ -26,6 +26,7 @@ ) from temporalio.nexus.system.stream_service.client import Stream from temporalio.nexus.system.stream_service.workflow import create_stream +from temporalio.worker import Replayer from tests.helpers import new_worker _REDIS_URL = os.environ.get("STREAMS_REDIS_URL") @@ -172,13 +173,76 @@ async def test_workflow_appends_and_closes(client: Client): assert record.producer.id == f"wf:{run_id}" assert isinstance(record.producer.kind, ProducerKindWorkflow) assert record.producer.kind.value.run_id == run_id - assert record.producer.kind.value.epoch is not None + epoch = record.producer.kind.value.epoch + assert epoch is not None + assert epoch.started_event_id > 0 + assert epoch.attempt == 1 described = await stream.describe() assert described.status == StreamStatus.StreamStatusClosed err = await _stream_error(stream.append("c")) assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_CLOSED +@activity.defn +async def append_in_two_steps(stream_id: str, seen_id: str) -> None: + client = activity.client() + stream = client.get_stream(StreamRef(id=stream_id), body_type=str) + seen = client.get_stream(StreamRef(id=seen_id), body_type=str) + await stream.append("a", "b") + # The second step waits until the Workflow says that it has the first, so the + # Workflow reads while the Activity runs. + async for _ in seen.iter_read_items(wait=timedelta(seconds=30)): + break + await stream.append("c", "d") + await stream.close() + + +@workflow.defn +class ReadWhileActivityRunsWorkflow: + @workflow.run + async def run(self, stream_id: str, seen_id: str) -> list[str]: + writer = workflow.start_activity( + append_in_two_steps, + args=[stream_id, seen_id], + start_to_close_timeout=timedelta(seconds=60), + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + seen = workflow.get_stream(StreamRef(id=seen_id), body_type=str) + got: list[str] = [] + # The wait is far longer than the pause between the two steps, so the loop ends + # only with the page that is done. + async for page in stream.iter_read(wait=timedelta(seconds=30)): + for record in page.records: + got.append(record.body) + if record.body == "b": + await seen.append("the first step") + if page.done: + got.append("done") + await writer + return got + + +async def test_workflow_read_of_an_activity_on_redis(client: Client): + stream_client = _with_store(client, _store()) + stream = await _create_stream(stream_client) + seen = await _create_stream(stream_client) + async with new_worker( + stream_client, ReadWhileActivityRunsWorkflow, activities=[append_in_two_steps] + ) as worker: + handle = await stream_client.start_workflow( + ReadWhileActivityRunsWorkflow.run, + args=[stream.ref.id, seen.ref.id], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await handle.result() == ["a", "b", "c", "d", "done"] + # The default replay source keeps no Record in History, so the replay reads Redis. + await Replayer( + workflows=[ReadWhileActivityRunsWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + async def test_activity_appends_with_activity_producer(client: Client): stream_client = _with_store(client, _store()) stream = await _create_stream(stream_client) diff --git a/tests/contrib/streams/test_appender_workflow.py b/tests/contrib/streams/test_appender_workflow.py index 998149b2e..7a65e8510 100644 --- a/tests/contrib/streams/test_appender_workflow.py +++ b/tests/contrib/streams/test_appender_workflow.py @@ -5,6 +5,7 @@ from datetime import timedelta from typing import Any +import pytest from google.protobuf.message import Message import temporalio.api.streamservice.v1 @@ -177,12 +178,20 @@ async def test_replay_of_workflow_appender_writes_nothing(client: Client): stream_id = f"stream-{uuid.uuid4()}" handle, _ = await _run(stream_client, InFlightWorkflow, stream_id) - replay_store = BatchStore() + # A replay of an append that Core made needs a Store that runs on the bridge, as the + # first run had. A new memory Store shows any write, since it has no Stream. + replay_store = MemoryStreamStore() await Replayer( workflows=[InFlightWorkflow], stream_config=_with_store(client, replay_store).config()["stream_config"], ).replay_workflow(await handle.fetch_history()) - assert replay_store.operations == [] + with pytest.raises(StreamError) as err: + await ( + _with_store(client, replay_store) + .get_stream(StreamRef(id=stream_id), body_type=str) + .describe() + ) + assert err.value.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND assert await _bodies(stream_client, stream_id) == ["a", "b", "c", "d"] diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py new file mode 100644 index 000000000..52a21d08a --- /dev/null +++ b/tests/nexus/test_stream_direct.py @@ -0,0 +1,2833 @@ +"""Reads and appends of a Stream that Core makes inside the Workflow Task.""" + +from __future__ import annotations + +import asyncio +import os +import urllib.parse +import uuid +from collections.abc import AsyncGenerator, Sequence +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Self + +import pytest + +from temporalio import activity, workflow +from temporalio.api.common.v1 import Payload +from temporalio.api.enums.v1 import EventType, StreamErrorReason +from temporalio.bridge.proto.external_data import ( + StreamCloseEntry, + StreamMarkerData, + StreamReadEntry, +) +from temporalio.client import ( + Client, + StreamConfig, + StreamReplaySource, + WorkflowExecutionStatus, + WorkflowFailureError, + WorkflowHandle, + WorkflowHistory, +) +from temporalio.common import RawValue +from temporalio.converter import ( + DataConverter, + PayloadCodec, + SerializationContext, + WithSerializationContext, + WorkflowSerializationContext, +) +from temporalio.exceptions import ApplicationError, StreamError, StreamReplayError +from temporalio.nexus.system.stream_service import ( + Producer, + ProducerKindWorkflow, + StreamRecord, + StreamRef, + StreamStatus, +) +from temporalio.runtime import MetricBuffer, Runtime, TelemetryConfig +from temporalio.testing import MemoryStreamStore, WorkflowEnvironment +from temporalio.worker import ( + Interceptor, + Replayer, + StartNexusOperationInput, + Worker, + WorkflowInboundInterceptor, + WorkflowInterceptorClassInput, + WorkflowOutboundInterceptor, +) +from tests.helpers import assert_eventually, assert_task_fail_eventually, new_worker +from tests.helpers.streams import DictStreamStore +from tests.test_converter import SimpleCodec + +# A Record body that a test can look for in the bytes of History. +_MARK = "direct-read-mark" + + +def _stream_client( + client: Client, + store: Any = None, + *, + replay_from: StreamReplaySource = StreamReplaySource.STORE, + codec: bool = False, +) -> Client: + config = client.config() + config["stream_config"] = StreamConfig( + stores=[store if store is not None else MemoryStreamStore()], + workflow_read_replay_from=replay_from, + ) + if codec: + config["data_converter"] = DataConverter(payload_codec=SimpleCodec()) + return Client(**config) + + +def _markers(history: WorkflowHistory) -> list[str]: + return [ + event.marker_recorded_event_attributes.marker_name + for event in history.events + if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + ] + + +def _stream_markers_per_task(history: WorkflowHistory) -> list[int]: + """Give the number of `core_stream` markers that each completed Workflow Task wrote.""" + counts: list[int] = [] + for event in history.events: + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED: + counts.append(0) + elif ( + event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + and event.marker_recorded_event_attributes.marker_name == "core_stream" + ): + counts[-1] += 1 + return counts + + +def _completed_tasks(history: WorkflowHistory) -> int: + return sum( + 1 + for event in history.events + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED + ) + + +async def _filled_stream( + client: Client, bodies: list[str], *, close: bool = True +) -> StreamRef[str]: + stream = await client.create_stream(id=f"stream-{uuid.uuid4()}", body_type=str) + if bodies: + await stream.append(*bodies) + if close: + await stream.close() + return stream.ref + + +async def _read_all(client: Client, ref: StreamRef[str]) -> list[StreamRecord[str]]: + stream = client.get_stream(ref, body_type=str) + return [record async for record in stream.iter_read_items()] + + +async def _run( + client: Client, + workflow_type: type[Any], + *args: Any, + activities: list[Any] | None = None, + max_cached_workflows: int = 1000, + task_timeout: timedelta | None = None, +) -> tuple[WorkflowHandle[Any, Any], Any]: + async with new_worker( + client, + workflow_type, + activities=activities or [], + max_cached_workflows=max_cached_workflows, + ) as worker: + handle: WorkflowHandle[Any, Any] = await client.start_workflow( + workflow_type.run, + args=list(args), + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + task_timeout=task_timeout, + ) + return handle, await handle.result() + + +# A generator over `iter_read_items` that reads the Stream of an Activity that still runs. + + +@dataclass +class ModelCall: + chunks_id: str + seen_id: str + + +@activity.defn +async def stream_model_output(call: ModelCall) -> None: + client = activity.client() + chunks = client.get_stream(StreamRef(id=call.chunks_id), body_type=str) + seen = client.get_stream(StreamRef(id=call.seen_id), body_type=str) + await chunks.append("Hel", "lo") + # The Activity writes the rest only after the Workflow says that it has the first + # chunks. So the test passes only if Workflow code gets chunks while the Activity runs. + told = False + async for _ in seen.iter_read_items(wait=timedelta(seconds=30)): + told = True + break + if not told: + raise ApplicationError( + "The Workflow got no chunk while the Activity ran", non_retryable=True + ) + await chunks.append(", ", "world") + await chunks.close() + + +async def model_chunks(chunks_id: str) -> AsyncGenerator[str, None]: + """Yield each chunk of the model's Stream, as the model stub of a library does.""" + stream = workflow.get_stream(StreamRef(id=chunks_id), body_type=str) + # The wait is far longer than a pause of the model, so no page comes back empty. + async for record in stream.iter_read_items(wait=timedelta(seconds=30)): + yield record.body + + +@workflow.defn +class LiveModelWorkflow: + @workflow.run + async def run(self, call: ModelCall) -> str: + model = workflow.start_activity( + stream_model_output, call, start_to_close_timeout=timedelta(seconds=60) + ) + seen = workflow.get_stream(StreamRef(id=call.seen_id), body_type=str) + text = "" + async for chunk in model_chunks(call.chunks_id): + text += chunk + if text == "Hello": + await seen.append("the first chunks") + await model + return text + + +async def test_generator_over_iter_read_items_reads_a_running_activity( + client: Client, +): + stream_client = _stream_client(client) + chunks = await _filled_stream(stream_client, [], close=False) + seen = await _filled_stream(stream_client, [], close=False) + handle, text = await _run( + stream_client, + LiveModelWorkflow, + ModelCall(chunks.id, seen.id), + activities=[stream_model_output], + ) + assert text == "Hello, world" + history = await handle.fetch_history() + # The reads and the append write no Local Activity marker, and at most one marker for + # each Workflow Task. + assert "core_local_activity" not in _markers(history) + assert "core_stream" in _markers(history) + assert max(_stream_markers_per_task(history)) == 1 + + +# The pages of a read. + + +@dataclass +class ReadArgs: + stream_id: str + from_offset: str | None = None + wait_ms: int | None = None + max_records: int | None = None + max_bytes: int | None = None + store: str = "memory" + pages: int | None = None + """The most pages to read. ``None`` reads until the iterator ends.""" + + +@dataclass +class ReadResult: + bodies: list[str] + offsets: list[str] + page_sizes: list[int] + last_offsets: list[str | None] + dones: list[bool] + + +def _ms(value: int | None) -> timedelta | None: + return None if value is None else timedelta(milliseconds=value) + + +def _hex(value: bytes | None) -> str | None: + return None if value is None else value.hex() + + +@workflow.defn +class ReadPagesWorkflow: + @workflow.run + async def run(self, args: ReadArgs) -> ReadResult: + # The ref has its Store, so a replay with no Stream config knows it. + ref = StreamRef[str](id=args.stream_id, store=args.store) + stream = workflow.get_stream(ref, body_type=str) + result = ReadResult([], [], [], [], []) + async for page in stream.iter_read( + from_offset=( + bytes.fromhex(args.from_offset) + if args.from_offset is not None + else None + ), + max_records=args.max_records, + wait=_ms(args.wait_ms), + max_bytes=args.max_bytes, + ): + result.bodies.extend(record.body for record in page.records) + result.offsets.extend(record.offset.hex() for record in page.records) + result.page_sizes.append(len(page.records)) + result.last_offsets.append(_hex(page.last_offset)) + result.dones.append(page.done) + if len(result.page_sizes) == args.pages: + break + return result + + +async def _read_pages( + client: Client, args: ReadArgs, *, max_cached_workflows: int = 1000 +) -> ReadResult: + _, result = await _run( + client, ReadPagesWorkflow, args, max_cached_workflows=max_cached_workflows + ) + return result + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_closed_stream_ends_the_iterator_with_a_done_page( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + result = await _read_pages( + stream_client, + ReadArgs(ref.id, wait_ms=30_000), + max_cached_workflows=max_cached_workflows, + ) + assert result.bodies == ["a", "b", "c"] + # The iterator ends after the page with `done`, also with a `wait`. + assert result.dones[-1] is True + assert not any(result.dones[:-1]) + assert result.last_offsets[-1] == result.offsets[-1] + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_read_with_max_records_gives_pages_of_two_two_and_one( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c", "d", "e"]) + result = await _read_pages( + stream_client, + ReadArgs(ref.id, max_records=2), + max_cached_workflows=max_cached_workflows, + ) + assert result.bodies == ["a", "b", "c", "d", "e"] + assert result.page_sizes == [2, 2, 1] + assert result.dones == [False, False, True] + # Each page ends at its last Record, and the next one starts after it. + assert result.last_offsets == [ + result.offsets[1], + result.offsets[3], + result.offsets[4], + ] + + +@workflow.defn +class ReadOnePageWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str], wait_ms: int | None) -> list[Any]: + stream = workflow.get_stream(ref, body_type=str) + page = await stream.read(wait=_ms(wait_ms)) + return [[record.body for record in page.records], page.done, page.last_offset] + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_read_with_wait_on_a_quiet_stream_gives_an_empty_page( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run( + stream_client, + ReadOnePageWorkflow, + ref, + 1000, + max_cached_workflows=max_cached_workflows, + ) + assert result == [[], False, None] + # The page comes only after the wait. + events = (await handle.fetch_history()).events + elapsed = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert elapsed >= timedelta(seconds=1) + + +async def test_read_with_no_wait_on_an_empty_stream_gives_an_empty_page( + client: Client, +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run(stream_client, ReadOnePageWorkflow, ref, None) + assert result == [[], False, None] + events = (await handle.fetch_history()).events + # An empty Stream gives no reason to wait. + took = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert took < timedelta(seconds=5) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_iter_read_items_with_no_wait_gives_what_is_there_and_ends( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + handle, bodies = await _run( + stream_client, + ReadItemsWorkflow, + ref, + None, + max_cached_workflows=max_cached_workflows, + ) + assert bodies == ["a", "b", "c"] + events = (await handle.fetch_history()).events + # The read doesn't wait for the Store, so the run ends long before the first heartbeat. + took = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert took < timedelta(seconds=5) + + +@workflow.defn +class ReadItemsWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str], wait_ms: int | None) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + return [ + record.body async for record in stream.iter_read_items(wait=_ms(wait_ms)) + ] + + +async def test_page_that_takes_the_last_record_with_a_cap_is_done(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + # The cap takes the last Record of the closed Stream, so the page is done. + by_records = await _read_pages(stream_client, ReadArgs(ref.id, max_records=3)) + assert (by_records.page_sizes, by_records.dones) == ([3], [True]) + one = await _filled_stream(stream_client, ["a"]) + by_bytes = await _read_pages(stream_client, ReadArgs(one.id, max_bytes=1)) + assert (by_bytes.page_sizes, by_bytes.dones) == ([1], [True]) + # On an open Stream the same page isn't done. + open_ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + capped = await _read_pages( + stream_client, ReadArgs(open_ref.id, max_records=3, pages=1) + ) + assert (capped.page_sizes, capped.dones) == ([3], [False]) + + +async def test_page_with_max_bytes_gives_the_first_record_over_the_cap( + client: Client, +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + result = await _read_pages(stream_client, ReadArgs(ref.id, max_bytes=1)) + # The first Record of each page always comes, even over the cap. + assert result.page_sizes == [1, 1, 1] + assert result.dones == [False, False, True] + + +@activity.defn +async def append_after_a_quiet_gap(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + await asyncio.sleep(2) + await stream.append("b") + await stream.close() + + +@workflow.defn +class QuietGapWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + writer = workflow.start_activity( + append_after_a_quiet_gap, + stream_id, + start_to_close_timeout=timedelta(seconds=30), + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [ + record.body + async for record in stream.iter_read_items(wait=timedelta(seconds=10)) + ] + await writer + return got + + +async def test_iter_read_items_with_a_wait_reads_across_a_quiet_gap(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run( + stream_client, + QuietGapWorkflow, + ref.id, + activities=[append_after_a_quiet_gap], + ) + assert result == ["a", "b"] + # The gap of 2 s is shorter than the wait, so one Core read gets both Records. + assert len(_read_seqs(await handle.fetch_history())) == 1 + + +# A loop with no await in its body, and one that awaits for each Record. + + +def _read_entries(history: WorkflowHistory) -> list[list[StreamReadEntry]]: + """Give the read entries of the `core_stream` marker of each Workflow Task.""" + entries: list[list[StreamReadEntry]] = [] + for event in history.events: + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + entries.append(list(StreamMarkerData.FromString(payload.data).reads)) + return entries + + +def _read_seqs(history: WorkflowHistory) -> set[int]: + """Give the seq of each Core read that History has.""" + return {entry.seq for task in _read_entries(history) for entry in task} + + +@activity.defn +async def append_five_slowly(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + for index in range(5): + await asyncio.sleep(0.3) + await stream.append(f"record-{index}") + await stream.close() + + +@workflow.defn +class LoopWithNoAwaitWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + writer = workflow.start_activity( + append_five_slowly, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [] + async for record in stream.iter_read_items(wait=timedelta(seconds=10)): + got.append(record.body) + await writer + return got + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_loop_with_no_await_keeps_one_core_read( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, got = await _run( + stream_client, + LoopWithNoAwaitWorkflow, + ref.id, + activities=[append_five_slowly], + max_cached_workflows=max_cached_workflows, + ) + assert got == [f"record-{index}" for index in range(5)] + entries = _read_entries(await handle.fetch_history()) + # The loop asks for the next page in the activation of the last one, so one Core + # read gives every page. + assert len({entry.seq for task in entries for entry in task}) == 1 + assert any(len(task) == 1 and len(task[0].page_counts) > 1 for task in entries) + + +@workflow.defn +class LoopWithAppendWorkflow: + @workflow.run + async def run(self, stream_id: str, out_id: str) -> list[str]: + writer = workflow.start_activity( + append_five_slowly, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + out = workflow.get_stream(StreamRef(id=out_id), body_type=str) + got = [] + async for record in stream.iter_read_items(wait=timedelta(seconds=10)): + got.append(record.body) + await out.append(record.body) + await writer + return got + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_loop_that_awaits_an_append_closes_and_starts_the_read( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + out_ref = await _filled_stream(stream_client, [], close=False) + handle, got = await _run( + stream_client, + LoopWithAppendWorkflow, + ref.id, + out_ref.id, + activities=[append_five_slowly], + max_cached_workflows=max_cached_workflows, + ) + bodies = [f"record-{index}" for index in range(5)] + assert got == bodies + # Each Record went to the other Stream once and in order, whatever the replays. + assert [record.body for record in await _read_all(stream_client, out_ref)] == bodies + # The append ends the activation with no call waiting on the read, so the read + # closes, and the next page starts another one. + assert len(_read_seqs(await handle.fetch_history())) > 1 + + +# The example of the design review: +# +# let mut i = 0; +# while let item = stream.next() { +# workflow.spawn(loop { yield_once.await; i++; }) +# if i == 2 { await execute_activity(...); } +# } +# +# Each Record starts one more task that counts the wakes of the Workflow. So whether the +# Activity starts, and at which Record, depends on the activations that gave the Workflow +# its Records, and not on the Records. + + +class _Wakes: + """Lets a task of a Workflow wait for the next wake of the Workflow. + + Python has no call for it. ``asyncio.sleep(0)`` resumes in the same activation, so a + loop of it never lets the activation end. + """ + + def __init__(self) -> None: + self._new: list[asyncio.Future[None]] = [] + self._parked: list[asyncio.Future[None]] = [] + self._resumed = False + self._watch = asyncio.create_task(workflow.wait_condition(self._on_idle)) + + def _on_idle(self) -> bool: + # The event loop checks a condition each time it has no ready task. That is once at + # the end of a wake, and once more when the tasks that this resumed have run. The + # second check is in the same wake, so it resumes nothing. + if self._resumed: + self._resumed = False + self._parked += self._new + else: + self._resumed = bool(self._parked) + for future in self._parked: + if not future.done(): + future.set_result(None) + self._parked = self._new + self._new = [] + return False + + async def yield_once(self) -> None: + future: asyncio.Future[None] = asyncio.get_running_loop().create_future() + self._new.append(future) + await future + + +@workflow.defn +class CounterTasksWorkflow: + def __init__(self) -> None: + self.i = 0 + + async def count(self, wakes: _Wakes) -> None: + while True: + await wakes.yield_once() + self.i += 1 + + @workflow.run + async def run(self, stream_id: str) -> list[int]: + wakes = _Wakes() + seen: list[int] = [] + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + async for item in stream.iter_read_items(wait=timedelta(seconds=30)): + asyncio.create_task(self.count(wakes)) + seen.append(self.i) + if self.i == 2: + await workflow.execute_activity( + tag, item.body, start_to_close_timeout=timedelta(seconds=10) + ) + return seen + + +async def _read_calls(count: int) -> None: + """Wait until Workflow code has made this many `read` calls.""" + + async def check() -> None: + assert [name for name, _ in _intercepted].count("ReadRecords") >= count + + await assert_eventually(check, interval=timedelta(milliseconds=20)) + + +async def _run_counter_tasks( + client: Client, pages: list[list[str]], *, max_cached_workflows: int = 1000 +) -> tuple[list[int], WorkflowHistory]: + """Run the example, and give it each page of Records in an activation of its own. + + The last page must make the Workflow start its Activity. + """ + _intercepted.clear() + stream = await client.create_stream(id=f"stream-{uuid.uuid4()}", body_type=str) + # The Records that the Stream has when the read starts come in one page. + await stream.append(*pages[0]) + async with new_worker( + client, + CounterTasksWorkflow, + activities=[tag], + interceptors=[StreamCallInterceptor()], + max_cached_workflows=max_cached_workflows, + ) as worker: + handle = await client.start_workflow( + CounterTasksWorkflow.run, + stream.ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + for calls, page in enumerate(pages[1:], start=2): + # The loop asks for its next page in the activation that gave it the page + # before. A Record that comes after that call is in a later activation. + await _read_calls(calls) + await stream.append(*page) + + async def activity_started() -> None: + assert _scheduled_activities(await handle.fetch_history()) == ["tag"] + + await assert_eventually(activity_started) + await stream.close() + seen = await handle.result() + return seen, await handle.fetch_history() + + +_COUNTER_PAGES = [["record-0", "record-1"], ["record-2"], ["record-3"]] + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_counter_tasks_start_the_activity_by_the_activations( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + seen, history = await _run_counter_tasks( + stream_client, _COUNTER_PAGES, max_cached_workflows=max_cached_workflows + ) + # The first two Records start a task each, and both tasks count the next wake. The loop + # reads `i` before the tasks of a wake run, so `i` is 2 at the fourth Record. + assert seen == [0, 0, 0, 2] + # The marker of the Workflow Task that started the Activity has the three pages, and + # each page is in an activation of its own. + [entry] = _read_entries(history)[0] + assert list(entry.page_counts) == [2, 1, 1] + assert all(gap > 0 for gap in entry.page_activations[1:]) + # A replay gives each page in the activation that got it, so `i` is the same at each + # Record. With no cache, the Workflow Tasks after the first one were such replays too. + config = stream_client.config()["stream_config"] + await Replayer( + workflows=[CounterTasksWorkflow], stream_config=config + ).replay_workflow(history) + + +async def test_counter_tasks_with_one_page_start_no_activity(client: Client): + stream_client = _stream_client(client) + bodies = [body for page in _COUNTER_PAGES for body in page] + ref = await _filled_stream(stream_client, bodies) + handle, seen = await _run( + stream_client, CounterTasksWorkflow, ref.id, activities=[tag] + ) + # With the same Records in one activation, no task counts a wake between two of them. + assert seen == [0, 0, 0, 0] + assert _scheduled_activities(await handle.fetch_history()) == [] + + +async def test_replay_with_other_activations_is_a_nondeterminism_error(client: Client): + stream_client = _stream_client(client) + seen, history = await _run_counter_tasks(stream_client, _COUNTER_PAGES) + assert seen == [0, 0, 0, 2] + # Put every page of a read into the activation of its first page. `i` then stays 0, and + # the replay does not start the Activity that History has. + for event in history.events: + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + data = StreamMarkerData.FromString(payload.data) + for entry in data.reads: + gaps = [0] * (len(entry.page_activations) - 1) + del entry.page_activations[1:] + entry.page_activations.extend(gaps) + payload.data = data.SerializeToString() + config = stream_client.config()["stream_config"] + replay = await Replayer( + workflows=[CounterTasksWorkflow], stream_config=config + ).replay_workflow(history, raise_on_replay_failure=False) + assert isinstance(replay.replay_failure, workflow.NondeterminismError) + + +# The Records of one page, taken in several activations. Core records the page +# when it gives it. An Activity between two Records is a later Workflow Task, +# and that task has no marker. + + +@workflow.defn +class ActivityForEachRecordWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [] + async for item in stream.iter_read_items(): + got.append( + await workflow.execute_activity( + tag, item.body, start_to_close_timeout=timedelta(seconds=10) + ) + ) + return got + + +@workflow.defn +class OneRecordPerReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [] + offset = None + while True: + page = await stream.read(from_offset=offset, max_records=1) + for record in page.records: + got.append( + await workflow.execute_activity( + tag, + record.body, + start_to_close_timeout=timedelta(seconds=10), + ) + ) + offset = page.last_offset + if page.done or not page.records: + return got + + +async def _one_page_across_activations( + client: Client, workflow_type: type[Any], *, max_cached_workflows: int +) -> None: + ref = await _filled_stream(client, ["a", "b", "c", "d"]) + handle, got = await _run( + client, + workflow_type, + ref.id, + activities=[tag], + max_cached_workflows=max_cached_workflows, + ) + assert got == [f"tagged {body}" for body in "abcd"] + history = await handle.fetch_history() + # The last Activity's result is a task of its own, and that task completes + # the Workflow. It records nothing, so the marker stays in the first task. + assert _completed_tasks(history) == 5 + assert _scheduled_activities(history) == ["tag", "tag", "tag", "tag"] + task = 0 + marker_task = None + for event in history.events: + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED: + task += 1 + elif ( + event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + and event.marker_recorded_event_attributes.marker_name == "core_stream" + ): + assert marker_task is None + marker_task = task + assert marker_task == 1 + entries = _read_entries(history) + assert len(entries) == 1 + [entry] = entries[0] + assert list(entry.page_counts) == [4] + assert list(entry.page_activations) == [1] + config = client.config()["stream_config"] + await Replayer(workflows=[workflow_type], stream_config=config).replay_workflow( + history + ) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_one_page_across_activities_keeps_its_marker( + client: Client, max_cached_workflows: int +): + await _one_page_across_activations( + _stream_client(client), + ActivityForEachRecordWorkflow, + max_cached_workflows=max_cached_workflows, + ) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_one_record_reads_of_one_page_keep_its_marker( + client: Client, max_cached_workflows: int +): + await _one_page_across_activations( + _stream_client(client), + OneRecordPerReadWorkflow, + max_cached_workflows=max_cached_workflows, + ) + + +# What `read` checks, and where it can't run. + + +# A client's read clamps each of these, or refuses it, and a Workflow read does the same. +_READ_LIMIT_CASES: list[dict[str, Any]] = [ + {"max_bytes": -1}, + {"max_records": 0}, + {"max_records": -1}, + {"max_records": 5000}, + {"wait": timedelta(seconds=-1)}, + {"wait": timedelta(seconds=120)}, +] + + +@workflow.defn +class ReadLimitsWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[Any]: + stream = workflow.get_stream(ref, body_type=str) + results: list[Any] = [] + for arguments in _READ_LIMIT_CASES: + try: + page = await stream.read(**arguments) + results.append(len(page.records)) + except StreamError as err: + results.append([StreamErrorReason.Name(err.reason), err.message]) + return results + + +async def test_read_takes_the_limits_of_a_client_read(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + _, results = await _run(stream_client, ReadLimitsWorkflow, ref) + stream = stream_client.get_stream(ref, body_type=str) + expected: list[Any] = [] + for arguments in _READ_LIMIT_CASES: + try: + page = await stream.read(**arguments) + expected.append(len(page.records)) + except StreamError as err: + expected.append([StreamErrorReason.Name(err.reason), err.message]) + assert results == expected + assert results[0] == ["STREAM_ERROR_REASON_INVALID", expected[0][1]] + + +@workflow.defn +class LongOffsetWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + got: list[str] = [] + try: + await stream.read(from_offset=b"o" * 2000) + got.append("page") + except StreamError as err: + got.append(StreamErrorReason.Name(err.reason)) + got.extend([record.body async for record in stream.iter_read_items()]) + return got + + +async def test_read_from_an_offset_over_1024_bytes_raises_invalid(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b"]) + handle, got = await _run(stream_client, LongOffsetWorkflow, ref) + assert got == ["STREAM_ERROR_REASON_INVALID", "a", "b"] + config = stream_client.config()["stream_config"] + await Replayer( + workflows=[LongOffsetWorkflow], stream_config=config + ).replay_workflow(history=await handle.fetch_history()) + + +@workflow.defn +class QueryReadWorkflow: + def __init__(self) -> None: + self.ref: StreamRef[str] | None = None + + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + self.ref = ref + await workflow.wait_condition(lambda: False) + + @workflow.query + def start_read(self) -> str: + assert self.ref is not None + step = workflow.get_stream(self.ref, body_type=str).read() + try: + step.send(None) + except Exception as err: + return type(err).__name__ + finally: + step.close() + return "started" + + +async def test_read_in_a_query_raises_the_read_only_error(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"]) + async with new_worker(stream_client, QueryReadWorkflow) as worker: + handle = await stream_client.start_workflow( + QueryReadWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await handle.query(QueryReadWorkflow.start_read) == ( + "ReadOnlyContextError" + ) + await handle.terminate() + + +# Reads that resume from an offset. + + +@workflow.defn +class LoopReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + got: list[str] = [] + # The first loop leaves with `break`, and its next request resumes there. + items = stream.iter_read_items(max_records=4) + async for record in items: + got.append(record.body) + if len(got) == 3: + break + assert items.next_request is not None + last = items.next_request.from_offset + # The last reads go on from there to the end, two Records at a time. + while True: + page = await stream.read(from_offset=last, max_records=2) + got.extend(record.body for record in page.records) + last = page.last_offset + if page.done: + return got + + +async def test_reads_from_an_offset_get_each_record_once(client: Client): + stream_client = _stream_client(client) + bodies = [f"record-{index}" for index in range(10)] + ref = await _filled_stream(stream_client, bodies) + _, got = await _run(stream_client, LoopReadWorkflow, ref) + assert got == bodies + + +# The marker in History. + + +async def _history_of_a_read( + client: Client, replay_from: StreamReplaySource +) -> tuple[WorkflowHistory, StreamRef[str], Client]: + stream_client = _stream_client(client, replay_from=replay_from) + ref = await _filled_stream(stream_client, [f"{_MARK}-1", f"{_MARK}-2"]) + handle, result = await _run(stream_client, ReadPagesWorkflow, ReadArgs(ref.id)) + assert result.bodies == [f"{_MARK}-1", f"{_MARK}-2"] + return await handle.fetch_history(), ref, stream_client + + +def _markers_have_mark(history: WorkflowHistory) -> bool: + # The result of the Workflow holds the bodies, so only the markers count. + return any( + _MARK.encode() in event.SerializeToString() + for event in history.events + if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + ) + + +async def test_store_marker_holds_no_body(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + assert "core_stream" in _markers(history) + assert not _markers_have_mark(history) + + +async def test_history_marker_holds_the_bodies(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.HISTORY) + assert "core_stream" in _markers(history) + assert _markers_have_mark(history) + + +# Replay. + + +async def test_replay_of_store_marker_reads_the_store(client: Client): + history, _, stream_client = await _history_of_a_read( + client, StreamReplaySource.STORE + ) + config = stream_client.config()["stream_config"] + await Replayer(workflows=[ReadPagesWorkflow], stream_config=config).replay_workflow( + history + ) + + +async def _failed_replay( + history: WorkflowHistory, ref: StreamRef[str], config: StreamConfig | None +) -> str: + """Replay with and without a raise, and give the text of the failure.""" + replayer = Replayer(workflows=[ReadPagesWorkflow], stream_config=config) + with pytest.raises(RuntimeError) as err: + await replayer.replay_workflow(history) + result = await replayer.replay_workflow(history, raise_on_replay_failure=False) + assert isinstance(result.replay_failure, RuntimeError) + assert str(result.replay_failure) == str(err.value) + message = str(err.value) + assert f"StreamReplayError: Stream `{ref.id}` in Store `memory`" in message + return message + + +async def test_replay_of_store_marker_with_no_stream_config_fails(client: Client): + history, ref, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + message = await _failed_replay(history, ref, None) + assert "this Worker has no Stream runtime for the Store `memory`" in message + + +async def test_replay_of_store_marker_fails_when_records_are_gone(client: Client): + history, ref, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + # A new memory Store of the same name has none of the Records. + message = await _failed_replay( + history, ref, StreamConfig(stores=[MemoryStreamStore()]) + ) + assert "stream_not_found" in message + + +async def test_replay_of_history_marker_needs_no_stream_config(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.HISTORY) + await Replayer(workflows=[ReadPagesWorkflow]).replay_workflow(history) + + +# Two reads at once, on a Worker with no cache and on a cached one. + + +@activity.defn +async def tag(body: str) -> str: + return f"tagged {body}" + + +@dataclass +class TwoReadsArgs: + first_id: str + second_id: str + out_id: str + + +@dataclass +class TwoReadsResult: + first: list[str] + second: list[str] + count: int + reads: int + + +@workflow.defn +class TwoReadsWorkflow: + def __init__(self) -> None: + self.count = 0 + self.reads = 0 + + async def _read(self, stream_id: str, out: Any) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got: list[str] = [] + last = None + while True: + self.reads += 1 + page = await stream.read(from_offset=last, max_records=2) + for record in page.records: + tagged = await workflow.execute_local_activity( + tag, record.body, start_to_close_timeout=timedelta(seconds=10) + ) + await out.append(tagged) + self.count += 1 + got.append(tagged) + last = page.last_offset + if page.done: + return got + # A timer ends the Workflow Task, so a Worker with no cache replays each read. + await workflow.sleep(0.05) + + @workflow.run + async def run(self, args: TwoReadsArgs) -> TwoReadsResult: + out = workflow.get_stream(StreamRef(id=args.out_id), body_type=str) + first, second = await asyncio.gather( + self._read(args.first_id, out), self._read(args.second_id, out) + ) + return TwoReadsResult(first, second, self.count, self.reads) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_two_reads_with_and_without_a_cache( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + first = [f"first-{index}" for index in range(5)] + second = [f"second-{index}" for index in range(5)] + out_ref = await _filled_stream(stream_client, [], close=False) + args = TwoReadsArgs( + first_id=(await _filled_stream(stream_client, first)).id, + second_id=(await _filled_stream(stream_client, second)).id, + out_id=out_ref.id, + ) + handle, result = await _run( + stream_client, + TwoReadsWorkflow, + args, + activities=[tag], + max_cached_workflows=max_cached_workflows, + ) + assert result == TwoReadsResult( + first=[f"tagged {body}" for body in first], + second=[f"tagged {body}" for body in second], + count=10, + reads=6, + ) + # Each Record went to the out Stream once, whatever the replays. + out = [record.body for record in await _read_all(stream_client, out_ref)] + assert sorted(out) == sorted(result.first + result.second) + assert _completed_tasks(await handle.fetch_history()) > 2 + + +# A Direct append. + + +@workflow.defn +class AppendWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str | None]: + stream = workflow.get_stream(ref, body_type=str) + appended = await stream.append("a", "b") + # A timer ends the Workflow Task, so a Worker with no cache replays the append. + await workflow.sleep(0.05) + return [_hex(appended.first_offset), _hex(appended.last_offset)] + + +async def test_direct_append(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, offsets = await _run( + stream_client, AppendWorkflow, ref, max_cached_workflows=0 + ) + records = await _read_all(stream_client, ref) + # The replay of the next Workflow Task appended nothing. + assert [record.body for record in records] == ["a", "b"] + assert offsets == [record.offset.hex() for record in records] + history = await handle.fetch_history() + run_id = handle.result_run_id + started = [ + event.event_id + for event in history.events + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_STARTED + ] + for record in records: + producer = record.producer + assert isinstance(producer, Producer) + assert producer.id == f"wf:{run_id}" + assert isinstance(producer.kind, ProducerKindWorkflow) + assert producer.kind.value.run_id == run_id + epoch = producer.kind.value.epoch + assert epoch is not None + # The epoch is the first Workflow Task, which made the append. + assert (epoch.started_event_id, epoch.attempt) == (started[0], 1) + assert _markers(history) == ["core_stream"] + # A Replayer appends nothing either. + await Replayer( + workflows=[AppendWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(history) + assert len(await _read_all(stream_client, ref)) == 2 + + +@workflow.defn +class RefusedAppendWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + try: + await stream.append("a") + except StreamError as err: + # A timer only on the error path, so a replay without the error differs. + await workflow.sleep(0.01) + return [ + StreamErrorReason.Name(err.reason), + err.message, + str(err.non_retryable), + ] + return ["appended"] + + +async def test_refused_append_raises_stream_error_live_and_in_replay(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"]) + handle, result = await _run(stream_client, RefusedAppendWorkflow, ref) + # The client path gives the same error for the same append. + with pytest.raises(StreamError) as client_err: + await stream_client.get_stream(ref, body_type=str).append("a") + assert result == [ + "STREAM_ERROR_REASON_CLOSED", + client_err.value.message, + "True", + ] + await Replayer( + workflows=[RefusedAppendWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +# A Workflow Task that fails once in the middle of a read. + +_fail_read_task = True + + +@workflow.defn(sandboxed=False) +class FailOnceInReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str], out_ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + out = workflow.get_stream(out_ref, body_type=str) + got: list[str] = [] + async for record in stream.iter_read_items(): + got.append(record.body) + if record.body == "record-2": + await out.append("seen record-2") + global _fail_read_task + if _fail_read_task: + _fail_read_task = False + raise RuntimeError("Failed the Workflow Task once") + return got + + +async def test_failed_task_in_a_read_gets_each_record_once(client: Client): + global _fail_read_task + _fail_read_task = True + stream_client = _stream_client(client) + bodies = [f"record-{index}" for index in range(5)] + ref = await _filled_stream(stream_client, bodies) + out_ref = await _filled_stream(stream_client, [], close=False) + _, got = await _run(stream_client, FailOnceInReadWorkflow, ref, out_ref) + assert not _fail_read_task + assert got == bodies + # The append of the failed task reached the Store, and its retry appends again. + out = await _read_all(stream_client, out_ref) + assert [record.body for record in out] == ["seen record-2"] * 2 + epochs = [] + for record in out: + assert record.producer is not None + assert isinstance(record.producer.kind, ProducerKindWorkflow) + epoch = record.producer.kind.value.epoch + assert epoch is not None + epochs.append((epoch.started_event_id, epoch.attempt)) + assert epochs[0] < epochs[1] + + +# A Signal while a read waits. + + +@workflow.defn +class SignalStopsReadWorkflow: + def __init__(self) -> None: + self.signaled = False + + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[Any]: + stream = workflow.get_stream(ref, body_type=str) + got: list[str] = [] + + async def read_all() -> None: + async for record in stream.iter_read_items(wait=timedelta(seconds=60)): + got.append(record.body) + + reader = asyncio.create_task(read_all()) + await workflow.wait_condition(lambda: self.signaled) + # The cancel leaves the read with no call that waits on it, so it closes. + reader.cancel() + try: + await reader + except asyncio.CancelledError: + pass + return [got, self.signaled] + + @workflow.signal + def stop(self) -> None: + self.signaled = True + + +async def test_signal_reaches_a_waiting_read(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, SignalStopsReadWorkflow) as worker: + handle = await stream_client.start_workflow( + SignalStopsReadWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + # The heartbeat of an open task comes at 80 % of its timeout. + task_timeout=timedelta(seconds=2), + ) + await asyncio.sleep(0.5) + await handle.signal(SignalStopsReadWorkflow.stop) + result = await asyncio.wait_for(handle.result(), timeout=30) + assert result == [["a"], True] + await Replayer( + workflows=[SignalStopsReadWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +# A second loop after a `break`, and the ends of a run with a read open. + + +@activity.defn +async def append_and_close(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("x", "y") + await stream.close() + + +@workflow.defn +class BreakThenReadAgainWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[Any]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + first = stream.iter_read_items(wait=timedelta(seconds=30)) + got_first: list[str] = [] + async for record in first: + got_first.append(record.body) + if len(got_first) == 2: + break + assert first.next_request is not None + # The second loop resumes after the last Record that the first one gave. + second = stream.iter_read( + from_offset=first.next_request.from_offset, wait=timedelta(seconds=30) + ) + writer = workflow.start_activity( + append_and_close, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + got_second: list[str] = [] + done = False + async for page in second: + got_second.extend(record.body for record in page.records) + done = page.done + await writer + return [got_first, got_second, done] + + +async def test_second_loop_after_a_break_gets_every_record(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + handle, result = await _run( + stream_client, BreakThenReadAgainWorkflow, ref.id, activities=[append_and_close] + ) + assert result == [["a", "b"], ["c", "x", "y"], True] + await Replayer( + workflows=[BreakThenReadAgainWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +@workflow.defn +class CompleteWithOpenReadsWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + first = await stream.read(wait=timedelta(seconds=60)) + # A second read waits in its own task while the Workflow returns. + waiting = asyncio.create_task( + stream.read(from_offset=first.last_offset, wait=timedelta(seconds=60)) + ) + await asyncio.sleep(0) + assert not waiting.done() + return "completed" + + +async def test_workflow_completes_with_open_reads(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, CompleteWithOpenReadsWorkflow) as worker: + handle = await stream_client.start_workflow( + CompleteWithOpenReadsWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await asyncio.wait_for(handle.result(), timeout=20) == "completed" + await Replayer( + workflows=[CompleteWithOpenReadsWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +@workflow.defn +class CancelDuringReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[Any]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got: list[str] = [] + try: + async for record in stream.iter_read_items(wait=timedelta(seconds=60)): + got.append(record.body) + except asyncio.CancelledError: + return [got, "cancelled"] + return [got, "ended"] + + +async def test_cancel_reaches_a_waiting_read(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, CancelDuringReadWorkflow) as worker: + handle = await stream_client.start_workflow( + CancelDuringReadWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + # The heartbeat of an open task comes at 80 % of its timeout. + task_timeout=timedelta(seconds=2), + ) + await asyncio.sleep(0.5) + await handle.cancel() + result = await asyncio.wait_for(handle.result(), timeout=30) + assert result == [["a"], "cancelled"] + + +@workflow.defn +class ContinueWithOpenReadWorkflow: + @workflow.run + async def run(self, stream_id: str, from_offset: str | None) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + if from_offset is None: + # The Core read of this page is still open when the run continues. + first = await stream.read(max_records=1, wait=timedelta(seconds=60)) + assert first.last_offset is not None + workflow.continue_as_new(args=[stream_id, first.last_offset.hex()]) + items = stream.iter_read_items(from_offset=bytes.fromhex(from_offset)) + return [record.body async for record in items] + + +async def test_continue_as_new_with_an_open_read(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + async with new_worker(stream_client, ContinueWithOpenReadWorkflow) as worker: + handle = await stream_client.start_workflow( + ContinueWithOpenReadWorkflow.run, + args=[ref.id, None], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + result = await asyncio.wait_for(handle.result(), timeout=20) + assert result == ["b", "c"] + first_run = stream_client.get_workflow_handle( + handle.id, run_id=handle.first_execution_run_id + ) + await Replayer( + workflows=[ContinueWithOpenReadWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await first_run.fetch_history()) + + +# A Store written in Python. + + +@workflow.defn +class PythonStoreAppendWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + await workflow.get_stream(ref, body_type=str).append("a") + + +async def test_python_store_appends_through_its_local_activity(client: Client): + stream_client = _stream_client(client, DictStreamStore()) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run(stream_client, PythonStoreAppendWorkflow, ref) + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + assert _markers(await handle.fetch_history()) == ["core_local_activity"] + + +async def test_python_store_refuses_a_read(client: Client): + stream_client = _stream_client(client, DictStreamStore()) + ref = await _filled_stream(stream_client, ["a"]) + with pytest.raises(WorkflowFailureError) as err: + await _run(stream_client, ReadPagesWorkflow, ReadArgs(ref.id, store="dict")) + cause = err.value.cause + assert isinstance(cause, ApplicationError) + assert cause.type == "StreamConfigError" + assert cause.non_retryable + assert cause.message == ( + "The Stream Store 'dict' has no Direct read, since it is written in Python. " + "Read the Stream in an Activity, or use a Store that runs on the bridge, such " + "as the Redis Store or MemoryStreamStore." + ) + + +# A payload codec. + + +@workflow.defn +class AppendThenReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + await stream.append("a", "b") + return [record.body async for record in stream.iter_read_items()] + + +async def test_payload_codec_covers_direct_append_and_read(client: Client): + store = MemoryStreamStore() + stream_client = _stream_client(client, store, codec=True) + ref = await _filled_stream(stream_client, [], close=False) + _, bodies = await _run(stream_client, AppendThenReadWorkflow, ref) + assert bodies == ["a", "b"] + # A client without the codec reads the bodies as the Store keeps them. + raw = _stream_client(client, store).get_stream( + StreamRef(id=ref.id), body_type=RawValue + ) + stored = [record.body.payload async for record in raw.iter_read_items()] + assert [body.metadata["simple-codec"] for body in stored] == [b"true", b"true"] + + +# A close after a Direct append. + + +@workflow.defn +class AppendThenCloseWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + stream = workflow.get_stream(ref, body_type=str) + await stream.append("a") + await stream.close() + + +async def test_close_after_direct_append(client: Client): + # The append writes as the Workflow with its task as the epoch. The close, from an + # Activity, sends the same Workflow Producer with no epoch, and the Store fences it. + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + await _run(stream_client, AppendThenCloseWorkflow, ref) + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + + +# A measurement. + + +@dataclass +class Burst: + stream_id: str + records: int = 200 + batches: int = 20 + over_ms: int = 2000 + + +@activity.defn +async def write_burst(burst: Burst) -> None: + stream = activity.client().get_stream(StreamRef(id=burst.stream_id), body_type=str) + size = burst.records // burst.batches + for batch in range(burst.batches): + await stream.append( + *(f"record-{batch * size + index}" for index in range(size)) + ) + await asyncio.sleep(burst.over_ms / 1000 / burst.batches) + await stream.close() + + +@workflow.defn +class MeasureWorkflow: + @workflow.run + async def run(self, burst: Burst) -> int: + writer = workflow.start_activity( + write_burst, burst, start_to_close_timeout=timedelta(seconds=60) + ) + stream = workflow.get_stream(StreamRef(id=burst.stream_id), body_type=str) + count = 0 + async for _ in stream.iter_read_items(wait=timedelta(seconds=30)): + count += 1 + await writer + return count + + +@pytest.mark.parametrize( + "replay_from", [StreamReplaySource.STORE, StreamReplaySource.HISTORY] +) +async def test_measure_read_of_200_records_over_2_seconds( + client: Client, replay_from: StreamReplaySource +): + stream_client = _stream_client(client, replay_from=replay_from) + ref = await _filled_stream(stream_client, [], close=False) + handle, count = await _run( + stream_client, MeasureWorkflow, Burst(ref.id), activities=[write_burst] + ) + assert count == 200 + history = await handle.fetch_history() + sizes = [ + event.marker_recorded_event_attributes.ByteSize() + for event in history.events + if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + ] + print( + f"\n{replay_from.name}: {_completed_tasks(history)} Workflow Tasks, " + f"{len(sizes)} markers, {sum(sizes)} marker bytes, sizes {sizes}" + ) + + +# Two loops from one offset, and calls on a read that fails. + + +@workflow.defn +class TwoLoopsWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[list[str]]: + stream = workflow.get_stream(ref, body_type=str) + + async def drain() -> list[str]: + return [record.body async for record in stream.iter_read_items()] + + # Both loops ask for the first page before either gets it, so each one takes a + # read of its own. + first = asyncio.create_task(drain()) + second = asyncio.create_task(drain()) + return [await first, await second] + + +async def test_two_loops_from_one_offset_each_get_every_record(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + handle, result = await _run(stream_client, TwoLoopsWorkflow, ref) + assert result == [["a", "b", "c"], ["a", "b", "c"]] + assert len(_read_seqs(await handle.fetch_history())) == 2 + + +@workflow.defn +class FailedReadsWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + + async def read() -> str: + try: + await stream.read(wait=timedelta(seconds=10)) + except StreamError as err: + return StreamErrorReason.Name(err.reason) + return "page" + + # Two calls wait before the failure comes, and a later call fails too. + first = asyncio.create_task(read()) + second = asyncio.create_task(read()) + return [await first, await second, await read()] + + +async def test_each_read_of_a_missing_stream_raises(client: Client): + stream_client = _stream_client(client) + ref: StreamRef[str] = StreamRef(id=f"stream-{uuid.uuid4()}") + _, result = await _run(stream_client, FailedReadsWorkflow, ref) + assert result == ["STREAM_ERROR_REASON_NOT_FOUND"] * 3 + + +# A codec that uses its serialization context. + + +@dataclass(frozen=True) +class ContextCodec(PayloadCodec, WithSerializationContext): + """Writes the Workflow id of its context into each payload, and decodes only its own.""" + + workflow_id: str = "" + + def with_context(self, context: SerializationContext) -> ContextCodec: + if isinstance(context, WorkflowSerializationContext): + return ContextCodec(context.workflow_id) + return ContextCodec("") + + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + return [ + Payload( + metadata={"encoding": b"binary/ctx", "ctx": self.workflow_id.encode()}, + data=payload.SerializeToString(), + ) + for payload in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + out = [] + for payload in payloads: + if payload.metadata.get("encoding") != b"binary/ctx": + out.append(payload) + continue + if payload.metadata["ctx"] != self.workflow_id.encode(): + raise ApplicationError( + f"Encoded for {payload.metadata['ctx']!r}, not " + f"{self.workflow_id!r}", + non_retryable=True, + ) + out.append(Payload.FromString(payload.data)) + return out + + +@workflow.defn +class AppendThenReadAllWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + await stream.append("from-workflow") + return [record.body async for record in stream.iter_read_items()] + + +@pytest.mark.parametrize("store_kind", ["memory", "dict"]) +async def test_codec_runs_with_no_context_on_stream_bodies( + client: Client, store_kind: str +): + store = MemoryStreamStore() if store_kind == "memory" else DictStreamStore() + config = client.config() + config["stream_config"] = StreamConfig(stores=[store]) + config["data_converter"] = DataConverter(payload_codec=ContextCodec()) + stream_client = Client(**config) + ref = await _filled_stream(stream_client, ["from-client"], close=False) + if store_kind == "dict": + # A Store written in Python has no Direct read, so only the append runs. + await _run(stream_client, AppendOneWorkflow, ref) + else: + _, bodies = await _run(stream_client, AppendThenReadAllWorkflow, ref) + assert bodies == ["from-client", "from-workflow"] + # The client decodes with no context, so it reads the Workflow's body too. + assert [record.body for record in await _read_all(stream_client, ref)] == [ + "from-client", + "from-workflow", + ] + raw_config = client.config() + raw_config["stream_config"] = StreamConfig(stores=[store]) + raw = Client(**raw_config).get_stream(StreamRef(id=ref.id), body_type=RawValue) + stored = [record.body.payload async for record in raw.iter_read_items()] + assert [body.metadata["ctx"] for body in stored] == [b"", b""] + + +@workflow.defn +class AppendOneWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + await workflow.get_stream(ref, body_type=str).append("from-workflow") + + +# Replays with no Stream config of calls whose ref names no Store. + + +@workflow.defn +class UnnamedRefReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + return [record.body async for record in stream.iter_read_items()] + + +async def _unnamed_read_history( + client: Client, replay_from: StreamReplaySource +) -> tuple[WorkflowHistory, str]: + stream_client = _stream_client(client, replay_from=replay_from) + ref = await _filled_stream(stream_client, ["a", "b"]) + handle, result = await _run(stream_client, UnnamedRefReadWorkflow, ref.id) + assert result == ["a", "b"] + return await handle.fetch_history(), ref.id + + +async def test_history_replay_of_an_unnamed_ref_needs_no_config(client: Client): + history, _ = await _unnamed_read_history(client, StreamReplaySource.HISTORY) + await Replayer(workflows=[UnnamedRefReadWorkflow]).replay_workflow(history) + + +async def test_store_replay_of_an_unnamed_ref_with_no_config_fails(client: Client): + history, stream_id = await _unnamed_read_history(client, StreamReplaySource.STORE) + with pytest.raises(RuntimeError) as err: + await Replayer(workflows=[UnnamedRefReadWorkflow]).replay_workflow(history) + message = str(err.value) + assert f"StreamReplayError: Stream `{stream_id}` in Store `memory`" in message + assert "this Worker has no Stream runtime for the Store `memory`" in message + assert "Nondeterminism" not in message + + +@workflow.defn +class UnnamedRefAppendWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str | None: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + appended = await stream.append("a") + return _hex(appended.last_offset) + + +async def test_replay_of_an_unnamed_append_needs_no_config(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run(stream_client, UnnamedRefAppendWorkflow, ref.id) + history = await handle.fetch_history() + assert _markers(history) == ["core_stream"] + await Replayer(workflows=[UnnamedRefAppendWorkflow]).replay_workflow(history) + + +# A Record whose body fails to decode. + + +@dataclass +class Point: + x: int + + +@workflow.defn +class BadBodyWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[Any]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=Point) + first = await stream.read(max_records=1) + events: list[Any] = [["got", first.records[0].body.x]] + # The page of the bad body raises, and a read from the same offset gets it again. + for _ in range(2): + try: + await stream.read(from_offset=first.last_offset, max_records=1) + events.append(["page", "decoded"]) + except Exception as err: + events.append(["page", type(err).__name__]) + return events + + +async def test_page_with_a_body_that_fails_to_decode_raises_each_time( + client: Client, +): + stream_client = _stream_client(client) + stream = await stream_client.create_stream(id=f"s-{uuid.uuid4()}", body_type=object) + await stream.append({"x": 1}, "not a point", {"x": 3}) + await stream.close() + _, events = await _run(stream_client, BadBodyWorkflow, stream.ref.id) + assert events == [["got", 1], ["page", "TypeError"], ["page", "TypeError"]] + + +# A Worker with no Workflows and a Store with a bad option. + + +@activity.defn +async def noop() -> None: + pass + + +async def test_worker_without_workflows_builds_no_store_runtime(client: Client): + from temporalio.contrib.redis.streams import RedisStreamStore + + config = client.config() + config["stream_config"] = StreamConfig(stores=[RedisStreamStore("not a url")]) + # Such a Worker starts, as it did before Direct calls, since it gives Core no runtime. + async with Worker( + Client(**config), task_queue=f"tq-{uuid.uuid4()}", activities=[noop] + ): + pass + # A Worker with Workflows gives Core the runtime, so it builds it at once. + with pytest.raises(ValueError, match="The Redis URL is wrong"): + Worker( + Client(**config), + task_queue=f"tq-{uuid.uuid4()}", + workflows=[AppendOneWorkflow], + ) + + +# The Direct close. + + +def _close_entries(history: WorkflowHistory) -> list[StreamCloseEntry]: + """Give the close entries of the `core_stream` markers, in order.""" + entries: list[StreamCloseEntry] = [] + for event in history.events: + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + entries.extend(StreamMarkerData.FromString(payload.data).closes) + return entries + + +def _scheduled_activities(history: WorkflowHistory) -> list[str]: + return [ + event.activity_task_scheduled_event_attributes.activity_type.name + for event in history.events + if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED + ] + + +async def _status(client: Client, ref: StreamRef[str]) -> StreamStatus | None: + return (await client.get_stream(ref, body_type=str).describe()).status + + +@workflow.defn +class AppendMaybeCloseWorkflow: + @workflow.run + async def run(self, stream_id: str, close: bool) -> None: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + if close: + await stream.close() + + +async def test_awaited_close_closes_in_one_more_task(client: Client): + stream_client = _stream_client(client) + closed_ref = await _filled_stream(stream_client, [], close=False) + open_ref = await _filled_stream(stream_client, [], close=False) + closed, _ = await _run(stream_client, AppendMaybeCloseWorkflow, closed_ref.id, True) + left_open, _ = await _run( + stream_client, AppendMaybeCloseWorkflow, open_ref.id, False + ) + assert await _status(stream_client, closed_ref) == StreamStatus.StreamStatusClosed + assert await _status(stream_client, open_ref) == StreamStatus.StreamStatusOpen + history = await closed.fetch_history() + assert _scheduled_activities(history) == [] + assert "core_local_activity" not in _markers(history) + # The task of the close records its intent, and the next one gives the result. + assert _completed_tasks(history) == ( + _completed_tasks(await left_open.fetch_history()) + 1 + ) + intent, result = _close_entries(history) + assert intent.HasField("intent_epoch") and not intent.resolved + assert result.resolved and not result.HasField("failure") + + +@workflow.defn +class FinallyCloseWorkflow: + @workflow.run + async def run(self, stream_id: str, fail: bool) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + try: + await stream.append("a") + if fail: + raise ApplicationError("failed on purpose", non_retryable=True) + return "returned" + finally: + await stream.close() + + +@pytest.mark.parametrize("fail", [False, True]) +async def test_close_in_finally(client: Client, fail: bool): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + if fail: + with pytest.raises(WorkflowFailureError) as err: + await _run(stream_client, FinallyCloseWorkflow, ref.id, True) + assert isinstance(err.value.cause, ApplicationError) + assert err.value.cause.message == "failed on purpose" + else: + _, result = await _run(stream_client, FinallyCloseWorkflow, ref.id, False) + assert result == "returned" + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + + +@workflow.defn +class AppendAfterCloseWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + await stream.close() + try: + await stream.append("b") + except StreamError as err: + return [StreamErrorReason.Name(err.reason), str(err.non_retryable)] + return ["appended"] + + +async def test_append_after_the_close_is_refused(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + _, result = await _run(stream_client, AppendAfterCloseWorkflow, ref.id) + assert result == ["STREAM_ERROR_REASON_CLOSED", "True"] + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + + +_fail_close_task = True + + +@workflow.defn(sandboxed=False) +class FailAfterCloseOnceWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + close = asyncio.create_task(stream.close()) + # A few turns of the loop let the close send its command in this activation. + for _ in range(10): + await asyncio.sleep(0) + global _fail_close_task + if _fail_close_task: + _fail_close_task = False + raise RuntimeError("Failed the Workflow Task once") + await close + + +async def test_failed_task_after_the_close_leaves_the_stream_open(client: Client): + global _fail_close_task + _fail_close_task = True + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run(stream_client, FailAfterCloseOnceWorkflow, ref.id) + assert not _fail_close_task + # The retry could append, so the failed task closed nothing. + assert [record.body for record in await _read_all(stream_client, ref)] == ["a", "a"] + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + # History has one intent and one result, from the retry. + entries = _close_entries(await handle.fetch_history()) + assert [entry.resolved for entry in entries] == [False, True] + + +@workflow.defn +class CloseTwiceWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.close() + await stream.close() + return "closed" + + +async def test_close_of_a_closed_stream_succeeds(client: Client): + stream_client = _stream_client(client) + # A client closed this Stream before the Workflow closes it twice. + ref = await _filled_stream(stream_client, ["a"]) + _, result = await _run(stream_client, CloseTwiceWorkflow, ref.id) + assert result == "closed" + open_ref = await _filled_stream(stream_client, ["a"], close=False) + _, result = await _run(stream_client, CloseTwiceWorkflow, open_ref.id) + assert result == "closed" + assert await _status(stream_client, open_ref) == StreamStatus.StreamStatusClosed + + +async def test_close_without_a_cache_and_in_replays(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run( + stream_client, AppendMaybeCloseWorkflow, ref.id, True, max_cached_workflows=0 + ) + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + history = await handle.fetch_history() + assert [entry.resolved for entry in _close_entries(history)] == [False, True] + # A Store with the same Stream, still open, shows that a replay closes nothing. + replay_store = MemoryStreamStore() + replay_client = _stream_client(client, replay_store) + await replay_client.create_stream(id=ref.id, body_type=str) + await Replayer( + workflows=[AppendMaybeCloseWorkflow], + stream_config=replay_client.config()["stream_config"], + ).replay_workflow(history) + assert await _status(replay_client, ref) == StreamStatus.StreamStatusOpen + # The ref names no Store, so a Replayer with no Stream config takes it from History. + await Replayer(workflows=[AppendMaybeCloseWorkflow]).replay_workflow(history) + + +@workflow.defn +class CloseNotAwaitedWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + # The task sends the close command, and the Workflow returns at once. + asyncio.create_task(stream.close()) + await asyncio.sleep(0) + return "returned" + + +async def test_close_not_awaited_closes_after_the_workflow_ends(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + handle, result = await _run(stream_client, CloseNotAwaitedWorkflow, ref.id) + assert result == "returned" + + async def closed() -> bool: + return await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + + for _ in range(50): + if await closed(): + break + await asyncio.sleep(0.1) + assert await closed() + # The task that ends the Workflow has the intent, and no task gives a result. + entries = _close_entries(await handle.fetch_history()) + assert [entry.resolved for entry in entries] == [False] + + +async def test_shutdown_waits_for_a_close_not_awaited(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, CloseNotAwaitedWorkflow) as worker: + result = await stream_client.execute_workflow( + CloseNotAwaitedWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert result == "returned" + # A graceful shutdown returns only after the Store answered the close in flight, so + # the Stream is closed with no wait here. + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + + +@activity.defn +async def sleep_a_while() -> None: + await asyncio.sleep(2) + + +@workflow.defn +class CancelDuringCloseWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + # The Local Activity holds the task of the close open, so the cancel comes in the + # task that gives the result of the close. + workflow.start_local_activity( + sleep_a_while, start_to_close_timeout=timedelta(seconds=10) + ) + try: + await stream.close() + except asyncio.CancelledError: + return "cancelled" + return "closed" + + +async def test_cancel_while_the_close_waits(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker( + stream_client, CancelDuringCloseWorkflow, activities=[sleep_a_while] + ) as worker: + handle = await stream_client.start_workflow( + CancelDuringCloseWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await asyncio.sleep(0.5) + await handle.cancel() + result = await asyncio.wait_for(handle.result(), timeout=30) + # The code gets the cancel, and Core still closed the Stream after the task of the + # intent. + assert result == "cancelled" + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + + +async def test_python_store_closes_through_its_local_activity(client: Client): + stream_client = _stream_client(client, DictStreamStore()) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run(stream_client, AppendMaybeCloseWorkflow, ref.id, True) + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + history = await handle.fetch_history() + assert _scheduled_activities(history) == [] + # One marker for the append and one for the close. + assert _markers(history) == ["core_local_activity"] * 2 + assert _close_entries(history) == [] + + +# The metrics of the Direct calls. + + +@workflow.defn +class ReadAppendCloseWorkflow: + @workflow.run + async def run(self, source_id: str, out_id: str) -> list[str]: + source = workflow.get_stream(StreamRef(id=source_id), body_type=str) + out = workflow.get_stream(StreamRef(id=out_id), body_type=str) + got = [record.body async for record in source.iter_read_items()] + await out.append(*got) + await out.close() + return got + + +async def test_direct_calls_record_their_metrics(env: WorkflowEnvironment): + buffer = MetricBuffer(10000) + runtime = Runtime(telemetry=TelemetryConfig(metrics=buffer)) + client = await env.connect_client(runtime=runtime) + store = MemoryStreamStore(name="metered", runtime=runtime) + stream_client = _stream_client(client, store) + source = await _filled_stream(stream_client, ["a", "b", "c"]) + out = await _filled_stream(stream_client, [], close=False) + async with new_worker(stream_client, ReadAppendCloseWorkflow) as worker: + result = await stream_client.execute_workflow( + ReadAppendCloseWorkflow.run, + args=[source.id, out.id], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert result == ["a", "b", "c"] + updates = [ + update + for update in buffer.retrieve_updates() + if update.metric.name.startswith("temporal_stream_") + ] + print("\nStream metrics:", sorted({update.metric.name for update in updates})) + + def values(name: str) -> list[Any]: + return [update.value for update in updates if update.metric.name == name] + + assert sum(values("temporal_stream_read_total")) == 1 + assert sum(values("temporal_stream_read_records")) == 3 + assert sum(values("temporal_stream_append_total")) == 1 + assert sum(values("temporal_stream_close_total")) == 1 + assert len(values("temporal_stream_append_latency")) == 1 + assert len(values("temporal_stream_close_latency")) == 1 + assert values("temporal_stream_read_failed") == [] + assert values("temporal_stream_append_failed") == [] + assert values("temporal_stream_close_failed") == [] + for update in updates: + assert update.attributes["stream_store"] == "metered" + assert update.attributes["workflow_type"] == "ReadAppendCloseWorkflow" + assert update.attributes["task_queue"] == worker.task_queue + + +# The example of the README: a Workflow starts each tool as its call arrives. + + +@dataclass +class ToolCall: + name: str + argument: str + + +@dataclass +class ModelEvent: + kind: str + text: str | None = None + tool_call: ToolCall | None = None + + +_MODEL_EVENTS = [ + ModelEvent(kind="text", text="Let me look."), + ModelEvent(kind="tool_call", tool_call=ToolCall(name="weather", argument="Oslo")), + ModelEvent(kind="text", text="And the time."), + ModelEvent(kind="tool_call", tool_call=ToolCall(name="clock", argument="Oslo")), + ModelEvent(kind="text", text="Done."), +] + + +@activity.defn +async def model_answer(events_id: str) -> None: + events = activity.client().get_stream(StreamRef(id=events_id), body_type=ModelEvent) + for event in _MODEL_EVENTS: + await events.append(event) + # The model takes a while between its events, so a tool can start before it ends. + await asyncio.sleep(0.5) + await events.close() + + +@activity.defn +async def run_tool(call: ToolCall) -> str: + return f"{call.name}({call.argument})" + + +@workflow.defn +class ToolCallsWorkflow: + @workflow.run + async def run(self, events_id: str, status_id: str) -> list[str]: + model = workflow.start_activity( + model_answer, events_id, start_to_close_timeout=timedelta(seconds=30) + ) + events = workflow.get_stream(StreamRef(id=events_id), body_type=ModelEvent) + status = workflow.get_stream(StreamRef(id=status_id), body_type=str) + tools = [] + async for record in events.iter_read_items(wait=timedelta(seconds=10)): + event = record.body + if event.kind == "tool_call": + assert event.tool_call is not None + # The tool starts now, before the model ends its answer. + tools.append( + workflow.start_activity( + run_tool, + event.tool_call, + start_to_close_timeout=timedelta(minutes=1), + ) + ) + await status.append(f"Running {event.tool_call.name}") + results = await asyncio.gather(*tools) + await model + return list(results) + + +async def test_workflow_starts_each_tool_as_its_call_arrives(client: Client): + stream_client = _stream_client(client) + events = await _filled_stream(stream_client, [], close=False) + status = await _filled_stream(stream_client, [], close=False) + handle, results = await _run( + stream_client, + ToolCallsWorkflow, + events.id, + status.id, + activities=[model_answer, run_tool], + ) + assert results == ["weather(Oslo)", "clock(Oslo)"] + assert [record.body for record in await _read_all(stream_client, status)] == [ + "Running weather", + "Running clock", + ] + # The first tool was scheduled while the model Activity still ran. + history = await handle.fetch_history() + scheduled: dict[int, str] = {} + first_tool = None + model_completed = None + for event in history.events: + if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED: + name = event.activity_task_scheduled_event_attributes.activity_type.name + scheduled[event.event_id] = name + if name == "run_tool" and first_tool is None: + first_tool = event.event_id + elif event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_COMPLETED: + attributes = event.activity_task_completed_event_attributes + if scheduled[attributes.scheduled_event_id] == "model_answer": + model_completed = event.event_id + assert first_tool is not None and model_completed is not None + assert first_tool < model_completed + + +# The read-ahead bound. + + +def test_read_ahead_bytes_below_one_raises(): + for bytes_ in (0, -1): + with pytest.raises(ValueError, match="workflow_read_ahead_bytes"): + StreamConfig(stores=[MemoryStreamStore()], workflow_read_ahead_bytes=bytes_) + + +async def test_read_with_a_read_ahead_of_one_byte_gives_every_record(client: Client): + config = client.config() + config["stream_config"] = StreamConfig( + stores=[MemoryStreamStore()], workflow_read_ahead_bytes=1 + ) + stream_client = Client(**config) + bodies = [f"record-{index}" for index in range(20)] + ref = await _filled_stream(stream_client, bodies) + _, result = await _run(stream_client, ReadPagesWorkflow, ReadArgs(ref.id)) + assert result.bodies == bodies + assert result.dones[-1] is True + + +# The cap on the Records of one page that Workflow code gets. + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_store_page_over_256_records_comes_in_pages_of_at_most_256( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + bodies = [f"record-{index}" for index in range(600)] + ref = await _filled_stream(stream_client, bodies) + handle, got = await _run( + stream_client, + ReadItemsWorkflow, + ref, + None, + max_cached_workflows=max_cached_workflows, + ) + assert got == bodies + history = await handle.fetch_history() + counts = [ + count + for task in _read_entries(history) + for entry in task + for count in entry.page_counts + ] + assert sum(counts) == 600 + assert max(counts) == 256 + config = stream_client.config()["stream_config"] + await Replayer(workflows=[ReadItemsWorkflow], stream_config=config).replay_workflow( + history + ) + + +# A replay that lost the Records of a read. + + +class _ReadThenWait: + def __init__(self) -> None: + self.go = False + + async def read_then_wait(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [record.body async for record in stream.iter_read_items()] + await workflow.wait_condition(lambda: self.go) + return got + + @workflow.signal + def proceed(self) -> None: + self.go = True + + +@workflow.defn +class ReadThenWaitWorkflow(_ReadThenWait): + @workflow.run + async def run(self, stream_id: str) -> list[str]: + return await self.read_then_wait(stream_id) + + +@workflow.defn(failure_exception_types=[StreamReplayError]) +class ReadThenWaitFailsWorkflow(_ReadThenWait): + @workflow.run + async def run(self, stream_id: str) -> list[str]: + return await self.read_then_wait(stream_id) + + +async def _replay_with_records_gone( + client: Client, workflow_type: type[Any], **worker_options: Any +) -> WorkflowHandle[Any, Any]: + """Read on one Worker, then give the run to a Worker whose Store lost the Records.""" + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b"]) + task_queue = f"tq-{uuid.uuid4()}" + # No cache, so the next Worker replays the History from the start. + async with new_worker( + stream_client, workflow_type, task_queue=task_queue, max_cached_workflows=0 + ): + handle: WorkflowHandle[Any, Any] = await stream_client.start_workflow( + workflow_type.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=task_queue, + ) + + async def read_is_in_history() -> None: + assert "core_stream" in _markers(await handle.fetch_history()) + + await assert_eventually(read_is_in_history) + # A new memory Store of the same name has none of the Records. + lost_client = _stream_client(client) + worker = new_worker( + lost_client, workflow_type, task_queue=task_queue, **worker_options + ) + async with worker: + await handle.signal("proceed") + if workflow_type is ReadThenWaitWorkflow and not worker_options: + await assert_task_fail_eventually( + handle, message_contains=f"StreamReplayError: Stream `{ref.id}`" + ) + await handle.terminate() + return handle + with pytest.raises(WorkflowFailureError) as err: + await asyncio.wait_for(handle.result(), timeout=30) + cause = err.value.cause + assert isinstance(cause, ApplicationError) + assert cause.type == "StreamReplayError" + assert f"StreamReplayError: Stream `{ref.id}`" in cause.message + return handle + + +async def test_lost_records_fail_the_task_by_default(client: Client): + await _replay_with_records_gone(client, ReadThenWaitWorkflow) + + +async def test_lost_records_fail_the_workflow_with_the_worker_option(client: Client): + await _replay_with_records_gone( + client, + ReadThenWaitWorkflow, + workflow_failure_exception_types=[StreamReplayError], + ) + + +async def test_lost_records_fail_the_workflow_with_the_defn_option(client: Client): + await _replay_with_records_gone(client, ReadThenWaitFailsWorkflow) + + +# A Workflow outbound interceptor sees each page that Workflow code reads. + +_intercepted: list[tuple[str, Any]] = [] + + +class StreamCallInterceptor(Interceptor): + def workflow_interceptor_class( + self, input: WorkflowInterceptorClassInput + ) -> type[WorkflowInboundInterceptor]: + return _StreamCallInbound + + +class _StreamCallInbound(WorkflowInboundInterceptor): + def init(self, outbound: WorkflowOutboundInterceptor) -> None: + super().init(_StreamCallOutbound(outbound)) + + +class _StreamCallOutbound(WorkflowOutboundInterceptor): + async def start_system_nexus_operation( + self, input: StartNexusOperationInput[Any, Any] + ) -> workflow.NexusOperationHandle[Any]: + _intercepted.append((input.operation_name, input.input)) + return await super().start_system_nexus_operation(input) + + +async def test_workflow_interceptor_sees_each_page_read(client: Client): + _intercepted.clear() + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c", "d", "e"]) + async with new_worker( + stream_client, ReadPagesWorkflow, interceptors=[StreamCallInterceptor()] + ) as worker: + result = await stream_client.execute_workflow( + ReadPagesWorkflow.run, + ReadArgs(ref.id, max_records=2), + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert result.page_sizes == [2, 2, 1] + assert [name for name, _ in _intercepted] == ["ReadRecords"] * 3 + # Each page starts after the last Record of the page before it. + assert [request.from_offset for _, request in _intercepted] == [ + None, + bytes.fromhex(result.offsets[1]), + bytes.fromhex(result.offsets[3]), + ] + + +# A Store outage that Core tries again, made with a proxy in front of Redis. + +_REDIS_URL = os.environ.get("STREAMS_REDIS_URL") + +_needs_redis = pytest.mark.skipif( + not _REDIS_URL, reason="STREAMS_REDIS_URL is not set to a Redis server" +) + + +async def _copy(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + except ConnectionError: + pass + + +class _StoreOutage: + """A TCP proxy in front of Redis that a test takes down and brings up. + + Down, it drops each open connection and each new one, so each call of the Store + fails with `StorageUnavailable`. Redis itself keeps running, so the tests that + share it in other processes see no outage. `refused` counts the connections that + it dropped while down. + """ + + def __init__(self, url: str) -> None: + self._parts = urllib.parse.urlsplit(url) + self._server: asyncio.Server | None = None + self._writers: set[asyncio.StreamWriter] = set() + self._down = False + self.url = "" + self.refused = 0 + + async def __aenter__(self) -> Self: + self._server = await asyncio.start_server(self._serve, "127.0.0.1", 0) + port = self._server.sockets[0].getsockname()[1] + credentials, at, _ = self._parts.netloc.rpartition("@") + netloc = f"{credentials}{at}127.0.0.1:{port}" + self.url = urllib.parse.urlunsplit(self._parts._replace(netloc=netloc)) + return self + + async def __aexit__(self, *_: object) -> None: + self.take_down() + if self._server: + self._server.close() + + def take_down(self) -> None: + self._down = True + for writer in list(self._writers): + writer.close() + + def bring_up(self) -> None: + self._down = False + + async def _serve( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + if self._down: + self.refused += 1 + writer.close() + return + try: + redis_reader, redis_writer = await asyncio.open_connection( + self._parts.hostname or "127.0.0.1", self._parts.port or 6379 + ) + except OSError: + writer.close() + return + self._writers.update((writer, redis_writer)) + copies = [ + asyncio.create_task(_copy(reader, redis_writer)), + asyncio.create_task(_copy(redis_reader, writer)), + ] + try: + await asyncio.wait(copies, return_when=asyncio.FIRST_COMPLETED) + finally: + for each in (writer, redis_writer): + self._writers.discard(each) + each.close() + await asyncio.gather(*copies) + + +def _redis_client(client: Client, url: str) -> Client: + from temporalio.contrib.redis.streams import RedisStreamStore + + # A short response timeout keeps a call that the outage cuts from waiting long. + store = RedisStreamStore( + url, + key_prefix=f"test-{uuid.uuid4()}", + response_timeout=timedelta(seconds=1), + ) + return _stream_client(client, store) + + +async def _store_answers(stream: Any) -> None: + # A client call has no retry, so the test waits until the Store answers again. + try: + await stream.describe() + except StreamError as err: + raise AssertionError(f"The Store doesn't answer: {err}") from err + + +@_needs_redis +async def test_append_during_a_short_outage_appends_once(client: Client): + assert _REDIS_URL + async with _StoreOutage(_REDIS_URL) as outage: + stream_client = _redis_client(client, outage.url) + ref = await _filled_stream(stream_client, [], close=False) + stream = stream_client.get_stream(ref, body_type=str) + async with new_worker( + stream_client, AppendWorkflow, max_cached_workflows=0 + ) as worker: + outage.take_down() + handle: WorkflowHandle[Any, Any] = await stream_client.start_workflow( + AppendWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + # The client gets the error that Core keeps from Workflow code. + with pytest.raises(StreamError) as err: + await stream.describe() + assert ( + err.value.reason + == StreamErrorReason.STREAM_ERROR_REASON_STORAGE_UNAVAILABLE + ) + + async def task_started() -> None: + events = (await handle.fetch_history()).events + assert any( + event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_STARTED + for event in events + ) + + await assert_eventually(task_started) + refused = outage.refused + await asyncio.sleep(1) + # The task waits for the append, which Core tries again. + assert outage.refused > refused + assert _completed_tasks(await handle.fetch_history()) == 0 + outage.bring_up() + offsets = await asyncio.wait_for(handle.result(), timeout=30) + records = await _read_all(stream_client, ref) + assert [record.body for record in records] == ["a", "b"] + assert offsets == [record.offset.hex() for record in records] + + +@workflow.defn +class ReadAcrossAnOutageWorkflow: + def __init__(self) -> None: + self._bodies: list[str] = [] + + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + async for record in stream.iter_read_items(wait=timedelta(seconds=10)): + self._bodies.append(record.body) + return self._bodies + + @workflow.query + def bodies(self) -> list[str]: + return self._bodies + + +@_needs_redis +async def test_read_across_a_short_outage_gives_each_record_once(client: Client): + assert _REDIS_URL + async with _StoreOutage(_REDIS_URL) as outage: + stream_client = _redis_client(client, outage.url) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + stream = stream_client.get_stream(ref, body_type=str) + async with new_worker(stream_client, ReadAcrossAnOutageWorkflow) as worker: + handle = await stream_client.start_workflow( + ReadAcrossAnOutageWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + + async def read_three() -> None: + bodies = await handle.query(ReadAcrossAnOutageWorkflow.bodies) + assert bodies == ["a", "b", "c"] + + await assert_eventually(read_three) + # The outage cuts the read that waits for the next Record. + outage.take_down() + await asyncio.sleep(1.5) + # Only the Worker's read asks the Store in the outage. + assert outage.refused > 0 + outage.bring_up() + await assert_eventually(lambda: _store_answers(stream)) + await stream.append("d", "e") + await stream.close() + result = await asyncio.wait_for(handle.result(), timeout=30) + assert result == ["a", "b", "c", "d", "e"] + + +@_needs_redis +@pytest.mark.parametrize("opt_in", ["worker", "defn"]) +async def test_replay_with_its_store_down_fails_only_the_task( + client: Client, opt_in: str +): + assert _REDIS_URL + workflow_type: type[Any] = ( + ReadThenWaitFailsWorkflow if opt_in == "defn" else ReadThenWaitWorkflow + ) + worker_options: dict[str, Any] = ( + {"workflow_failure_exception_types": [StreamReplayError]} + if opt_in == "worker" + else {} + ) + async with _StoreOutage(_REDIS_URL) as outage: + stream_client = _redis_client(client, outage.url) + ref = await _filled_stream(stream_client, ["a", "b"]) + task_queue = f"tq-{uuid.uuid4()}" + # No cache, so the next Worker replays the History and reads the Store. + async with new_worker( + stream_client, workflow_type, task_queue=task_queue, max_cached_workflows=0 + ): + handle: WorkflowHandle[Any, Any] = await stream_client.start_workflow( + workflow_type.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=task_queue, + ) + + async def read_is_in_history() -> None: + assert "core_stream" in _markers(await handle.fetch_history()) + + await assert_eventually(read_is_in_history) + outage.take_down() + async with new_worker( + stream_client, workflow_type, task_queue=task_queue, **worker_options + ): + await handle.signal("proceed") + + async def task_failed() -> None: + messages = [ + event.workflow_task_failed_event_attributes.failure.message + for event in (await handle.fetch_history()).events + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_FAILED + ] + assert messages + assert messages[0].startswith( + f"StreamReplayUnavailable: Stream `{ref.id}` in Store `redis`" + ), messages[0] + + # Core tries the Store for 5 s before it fails the task. + await assert_eventually(task_failed, timeout=timedelta(seconds=30)) + # The opt-in doesn't fail the Workflow, since the Records may still be there. + description = await handle.describe() + assert description.status == WorkflowExecutionStatus.RUNNING + outage.bring_up() + result = await asyncio.wait_for(handle.result(), timeout=60) + assert result == ["a", "b"] diff --git a/tests/nexus/test_stream_store_workflow.py b/tests/nexus/test_stream_store_workflow.py index 9bb641aa9..9b4b95c3c 100644 --- a/tests/nexus/test_stream_store_workflow.py +++ b/tests/nexus/test_stream_store_workflow.py @@ -104,7 +104,8 @@ async def run(self, stream_id: str) -> StreamRef[str]: # Workflow code appends either with the Workflow `Stream` or with a Local Activity of -# its own, and both write as the Local Activity that runs the append. +# its own. Core makes the first append in the Workflow Task as the Workflow, and the +# second writes as the Local Activity that runs it. _APPEND_WORKFLOWS = pytest.mark.parametrize( "append_workflow", [AppendWorkflow, LocalActivityAppendWorkflow] ) @@ -142,11 +143,10 @@ async def run(self, stream_id: str) -> None: @workflow.defn -class RefusedOperationWorkflow: +class NexusClientReadWorkflow: @workflow.run async def run(self, operation: str) -> str: - # The Workflow `Stream` has no `read`, so the test starts its operation through - # a Nexus client, as Workflow code still can. + # Workflow code can also start the operation of `read` through a Nexus client. requests: dict[str, Any] = { "ReadRecords": ReadRecordsRequest(ref=StreamRef[str](id="stream")), } @@ -154,11 +154,16 @@ async def run(self, operation: str) -> str: service=StreamService, endpoint=TEMPORAL_SYSTEM_ENDPOINT ) try: - await nexus_client.start_operation(operation, requests[operation]) + handle: workflow.NexusOperationHandle[ + Any + ] = await nexus_client.start_operation(operation, requests[operation]) + await handle + except StreamError as err: + return StreamErrorReason.Name(err.reason) except ApplicationError as err: assert err.non_retryable return err.message - return "started" + return "read" @workflow.defn @@ -333,10 +338,14 @@ async def append_records( class CloseRequestStore(RecordingStreamStore): - """Keeps each close request.""" + """Keeps each close request. + + It wraps a Store written in Python, so a close of Workflow code runs in its Local + Activity. + """ def __init__(self) -> None: - super().__init__(MemoryStreamStore()) + super().__init__(DictStreamStore()) self.closes: list[temporalio.api.streamservice.v1.CloseStreamRequest] = [] async def close_stream( @@ -390,6 +399,26 @@ async def run(self, stream_id: str) -> str: return "closed" +_fail_close_intent_task = True + + +@workflow.defn(sandboxed=False) +class CloseIntentThenFailTaskOnceWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + stream = get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + close = asyncio.create_task(stream.close()) + # A few turns of the loop let the close send its intent in this task. + for _ in range(10): + await asyncio.sleep(0) + global _fail_close_intent_task + if _fail_close_intent_task: + _fail_close_intent_task = False + raise RuntimeError("Failed the Workflow Task once") + await close + + class RecordingInterceptor(Interceptor): def __init__(self) -> None: self.activity_types: list[str] = [] @@ -581,8 +610,12 @@ async def test_workflow_stream_history_has_no_bodies( for event in events if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED ] - # One marker for the create and one for the append. - assert markers == ["core_local_activity", "core_local_activity"] + # One marker for the create and one for the append. Core writes its own marker for + # an append that it makes in the Workflow Task. + append_marker = ( + "core_stream" if append_workflow is AppendWorkflow else "core_local_activity" + ) + assert markers == ["core_local_activity", append_marker] for event in events: for body in _BODIES: assert body.encode() not in event.SerializeToString() @@ -593,13 +626,19 @@ async def test_workflow_stream_replay_does_not_call_store( client: Client, append_workflow: type[Any] ): stream_client = _with_store(client, MemoryStreamStore()) - handle, _ = await _run_append_workflow(stream_client, append_workflow) - replay_store = CountingStore("memory") + handle, ref = await _run_append_workflow(stream_client, append_workflow) + # A replay of an append that Core made needs a Store that runs on the bridge, as the + # first run had. A new memory Store shows any call, since it has no Stream. + replay_store = MemoryStreamStore() await Replayer( workflows=[append_workflow], stream_config=StreamConfig(stores=[replay_store]), ).replay_workflow(await handle.fetch_history()) - assert replay_store.operations == [] + with pytest.raises(StreamError) as err: + await ( + _with_store(client, replay_store).get_stream(ref, body_type=str).describe() + ) + assert err.value.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND async def test_workflow_create_without_stream_config_fails_the_workflow( @@ -630,25 +669,41 @@ async def test_workflow_create_without_stream_config_fails_the_workflow( ) -async def test_workflow_read_fails_with_store(client: Client): - stream_client = _with_store(client, MemoryStreamStore()) - async with new_worker(stream_client, RefusedOperationWorkflow) as worker: - result = await stream_client.execute_workflow( - RefusedOperationWorkflow.run, - "ReadRecords", - id=f"workflow-{uuid.uuid4()}", - task_queue=worker.task_queue, - ) - assert result == ( - "Workflow code can't read a Stream on a Store in this release. " - "Call an Activity that reads it through its client." +async def test_workflow_read_through_a_nexus_client_needs_a_runtime( + client: Client, +): + async def read_with(store: StreamStore) -> str: + stream_client = _with_store(client, store) + async with new_worker(stream_client, NexusClientReadWorkflow) as worker: + return await stream_client.execute_workflow( + NexusClientReadWorkflow.run, + "ReadRecords", + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + + # Core reads a Store with a Rust runtime, as for `read`. Nobody created the Stream. + assert await read_with(MemoryStreamStore()) == "STREAM_ERROR_REASON_NOT_FOUND" + assert await read_with(DictStreamStore()) == ( + "The Stream Store 'dict' has no Direct read, since it is written in Python. " + "Read the Stream in an Activity, or use a Store that runs on the bridge, such " + "as the Redis Store or MemoryStreamStore." ) -def test_workflow_stream_has_append_and_close_and_no_read(): - for name in ("append", "close", "describe", "__aenter__", "__aexit__"): +def test_workflow_stream_has_the_calls_of_a_client_stream(): + for name in ( + "append", + "close", + "describe", + "read", + "iter_read", + "iter_read_items", + "__aenter__", + "__aexit__", + ): assert hasattr(WorkflowStream, name) - assert not hasattr(WorkflowStream, "read") + assert not hasattr(WorkflowStream, "iter") async def test_workflow_stream_store_error_reaches_workflow_code(client: Client): @@ -785,8 +840,8 @@ async def test_workflow_task_retry_appends_with_higher_epoch( task_queue=worker.task_queue, ) records = await _read_all(stream_client, stream_id) - # The failed Workflow Task records no marker, so its retry runs the Local Activity - # again and writes a second copy. + # The failed Workflow Task records no marker, so its retry appends again and writes + # a second copy. assert [record.body for record in records] == [*_BODIES, *_BODIES] producers = _producers(records) assert len({producer.id for producer in producers}) == 1 @@ -1179,7 +1234,8 @@ async def test_failed_workflow_task_after_close_leaves_the_stream_closed( ): global _fail_close_task _fail_close_task = True - stream_client = _with_store(client, MemoryStreamStore()) + # A Store written in Python, since Core closes a Stream on a Store with a runtime. + stream_client = _with_store(client, RecordingStreamStore(MemoryStreamStore())) stream = await stream_client.create_stream( id=f"stream-{uuid.uuid4()}", body_type=str ) @@ -1207,9 +1263,35 @@ async def test_failed_workflow_task_after_close_leaves_the_stream_closed( ) +async def test_failed_workflow_task_after_direct_close_leaves_the_stream_open( + client: Client, +): + global _fail_close_intent_task + _fail_close_intent_task = True + stream_client = _with_store(client, MemoryStreamStore()) + stream = await stream_client.create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + async with new_worker(stream_client, CloseIntentThenFailTaskOnceWorkflow) as worker: + await stream_client.execute_workflow( + CloseIntentThenFailTaskOnceWorkflow.run, + stream.ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + # Core makes the close with no Activity. The failed task recorded no intent, so it + # closed nothing, and the retried task appends again before it closes the Stream. + records = await _read_all(stream_client, stream.ref.id) + assert [record.body for record in records] == ["a", "a"] + assert await _describe_status(stream_client, stream.ref.id) == ( + StreamStatus.StreamStatusClosed + ) + + async def test_activity_interceptors_see_the_close_activity(client: Client): interceptor = RecordingInterceptor() - stream_client = _with_store(client, MemoryStreamStore()) + # A Store written in Python, since Core closes a Stream on a Store with a runtime. + stream_client = _with_store(client, DictStreamStore()) async with new_worker( stream_client, CloseWorkflow, interceptors=[interceptor] ) as worker: @@ -1220,7 +1302,7 @@ async def test_activity_interceptors_see_the_close_activity(client: Client): task_queue=worker.task_queue, ) # The create, the append and the close each run as a Local Activity. - assert interceptor.activity_types == ["__temporal_stream_store_call:memory"] * 3 + assert interceptor.activity_types == ["__temporal_stream_store_call:dict"] * 3 assert interceptor.is_local == [True] * 3