Skip to content

Commit 2896da7

Browse files
committed
Make durable message streams portable across service mode
1 parent 319578e commit 2896da7

10 files changed

Lines changed: 591 additions & 3 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ Earlier SDK versions remain historical releases and are not alternate supported
1414
2.0 baselines; no prerelease compatibility shim is provided.
1515

1616
### Added
17+
- `WorkflowContext.message_stream()` now consumes named repeated input in
18+
ordered bounded batches. Runtime-owned cursor and wait metadata survives
19+
replay, worker replacement, server restart, duplicates, and continue-as-new
20+
without exposing the reserved transport signal to workflow declarations.
1721
- `WorkflowContext.upsert_memo()` now authors validated memo patches, replays
1822
`MemoUpserted` identities, carries opaque payload envelopes, and fails before
1923
completion when runtime capability discovery does not advertise memo updates.

‎pyproject.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ registry-version = "2.0.0rc36"
7373
supported-server-versions = "2.0.0-rc.50"
7474
worker-protocol-version = "1.16"
7575
control-plane-version = "2"
76+
message-streams = true
77+
message-streams-minimum-worker-protocol-version = "1.15"
7678
workflow-streams-contract = "durable-workflow.v2.workflow-streams.contract@1"
7779
workflow-streams-operations = "list,describe,subscribe,append,close,error"
7880
workflow-streams-authoring-command = "record_side_effect.workflow_stream"

‎src/durable_workflow/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,8 @@
212212
ActivityRetryPolicy,
213213
ChildWorkflowRetryPolicy,
214214
ContinueAsNew,
215+
MessageStream,
216+
MessageStreamMessage,
215217
NexusServiceCall,
216218
Replayer,
217219
ReplayOutcome,
@@ -236,6 +238,8 @@
236238
"ChildWorkflowTerminated",
237239
"Client",
238240
"ContinueAsNew",
241+
"MessageStream",
242+
"MessageStreamMessage",
239243
"NamespaceDescription",
240244
"NamespaceList",
241245
"NEXUS_CALLER_SDK_LANGUAGE",

‎src/durable_workflow/client.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@
6161

6262
PROTOCOL_VERSION = "1.16"
6363
CONTROL_PLANE_VERSION = "2"
64+
_MESSAGE_STREAMS_CAPABILITY = "message_streams"
65+
_MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL = (1, 15)
6466
CONTROL_PLANE_REQUEST_CONTRACT_SCHEMA = "durable-workflow.v2.control-plane-request.contract"
6567
CONTROL_PLANE_REQUEST_CONTRACT_VERSION = 1
6668
_QUERY_TASKS_DISCOVERY_PATH = "worker_protocol.server_capabilities.query_tasks"
@@ -128,6 +130,18 @@ def _protocol_version_from_env(name: str, default: str) -> str:
128130
return value.strip()
129131

130132

133+
def _worker_protocol_supports_message_streams() -> bool:
134+
version = _protocol_version_from_env(
135+
"DURABLE_WORKFLOW_WORKER_PROTOCOL_VERSION",
136+
PROTOCOL_VERSION,
137+
)
138+
parts = version.split(".")
139+
if len(parts) != 2 or not all(part.isdigit() for part in parts):
140+
return False
141+
142+
return (int(parts[0]), int(parts[1])) >= _MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL
143+
144+
131145
def _normalize_base_url(base_url: str) -> str:
132146
parsed = urlsplit(base_url)
133147
if parsed.query or parsed.fragment:
@@ -1240,6 +1254,20 @@ async def signal(self, signal_name: str, args: list[Any] | None = None) -> None:
12401254
"""Deliver an external signal to this workflow. See :meth:`Client.signal_workflow`."""
12411255
await self._client.signal_workflow(self.workflow_id, signal_name, args=args)
12421256

1257+
async def append_message(
1258+
self,
1259+
stream_name: str,
1260+
message_id: str,
1261+
args: list[Any] | None = None,
1262+
) -> dict[str, Any]:
1263+
"""Append one idempotently identified message to a durable input stream."""
1264+
return await self._client.append_message_stream(
1265+
self.workflow_id,
1266+
stream_name,
1267+
message_id,
1268+
args=args,
1269+
)
1270+
12431271
async def query(self, query_name: str, args: list[Any] | None = None) -> Any:
12441272
"""Execute a read-only query against this workflow. See :meth:`Client.query_workflow`."""
12451273
return await self._client.query_workflow(self.workflow_id, query_name, args=args)
@@ -3955,6 +3983,30 @@ async def signal_workflow(
39553983
)
39563984
await self._request("POST", f"/workflows/{workflow_id}/signal/{signal_name}", json=body, context=workflow_id)
39573985

3986+
async def append_message_stream(
3987+
self,
3988+
workflow_id: str,
3989+
stream_name: str,
3990+
message_id: str,
3991+
*,
3992+
args: list[Any] | None = None,
3993+
) -> dict[str, Any]:
3994+
"""Append repeated input without requiring workflow-authored cursor bookkeeping."""
3995+
body: dict[str, Any] = {"message_id": message_id}
3996+
if args is not None:
3997+
body["input"] = self._payload_envelope(
3998+
args,
3999+
kind="message_stream",
4000+
workflow_id=workflow_id,
4001+
)
4002+
result = await self._request(
4003+
"POST",
4004+
f"/workflows/{quote(workflow_id, safe='._:-')}/message-streams/{quote(stream_name, safe='._:-')}/messages",
4005+
json=body,
4006+
context=workflow_id,
4007+
)
4008+
return dict(result)
4009+
39584010
async def query_workflow(
39594011
self, workflow_id: str, query_name: str, *, args: list[Any] | None = None
39604012
) -> Any:
@@ -4532,6 +4584,12 @@ async def register_worker(
45324584
raise ValueError("max_concurrent_workflow_tasks must be at least 1")
45334585
if max_concurrent_activity_tasks is not None and max_concurrent_activity_tasks < 1:
45344586
raise ValueError("max_concurrent_activity_tasks must be at least 1")
4587+
if (
4588+
capabilities
4589+
and _MESSAGE_STREAMS_CAPABILITY in capabilities
4590+
and not _worker_protocol_supports_message_streams()
4591+
):
4592+
raise ValueError("message streams require worker protocol 1.15 or newer")
45354593

45364594
body: dict[str, Any] = {
45374595
"worker_id": worker_id,
@@ -4745,18 +4803,27 @@ async def complete_workflow_task(
47454803
lease_owner: str,
47464804
workflow_task_attempt: int,
47474805
commands: list[dict[str, Any]],
4806+
message_stream_cursors: list[dict[str, Any]] | None = None,
4807+
message_stream_waits: list[dict[str, Any]] | None = None,
47484808
) -> Any:
47494809
"""Report successful execution of a workflow task with its emitted commands.
47504810
47514811
Worker-plane endpoint, called by :class:`~durable_workflow.Worker`.
47524812
``commands`` is the list of serialized commands the workflow yielded
47534813
for this task.
47544814
"""
4815+
if (message_stream_cursors or message_stream_waits) and not _worker_protocol_supports_message_streams():
4816+
raise ValueError("message stream completion metadata requires worker protocol 1.15 or newer")
4817+
47554818
body: dict[str, Any] = {
47564819
"lease_owner": lease_owner,
47574820
"workflow_task_attempt": workflow_task_attempt,
47584821
"commands": commands,
47594822
}
4823+
if message_stream_cursors:
4824+
body["message_stream_cursors"] = message_stream_cursors
4825+
if message_stream_waits:
4826+
body["message_stream_waits"] = message_stream_waits
47604827
return await self._request(
47614828
"POST", f"/worker/workflow-tasks/{task_id}/complete", worker=True, json=body
47624829
)

‎src/durable_workflow/worker.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@
9191
QUERY_TASKS_CAPABILITY = "query_tasks"
9292
UPDATE_VALIDATION_TASKS_CAPABILITY = "update_validation_tasks"
9393
WORKFLOW_UPDATES_CAPABILITY = "workflow_updates"
94+
MESSAGE_STREAMS_CAPABILITY = "message_streams"
9495
MEMO_UPSERTS_CAPABILITY = "memo_upserts"
9596
TYPED_SEARCH_ATTRIBUTES_CAPABILITY = "typed_search_attributes"
9697

@@ -1131,6 +1132,7 @@ async def _register(self) -> None:
11311132
capabilities.append(UPDATE_VALIDATION_TASKS_CAPABILITY)
11321133
if any(contract["updates"] for contract in self.workflow_command_contracts.values()):
11331134
capabilities.append(WORKFLOW_UPDATES_CAPABILITY)
1135+
capabilities.append(MESSAGE_STREAMS_CAPABILITY)
11341136

11351137
ack = await self.client.register_worker(
11361138
worker_id=self.worker_id,
@@ -1539,6 +1541,8 @@ async def _run_workflow_task_core(self, task: dict[str, Any]) -> list[dict[str,
15391541
task_id=task_id,
15401542
attempt=attempt,
15411543
commands=commands,
1544+
message_stream_cursors=outcome.message_stream_cursors,
1545+
message_stream_waits=outcome.message_stream_waits,
15421546
)
15431547
except Exception as e:
15441548
log.warning("failed to complete workflow task %s: %s", task_id, e)
@@ -1553,6 +1557,8 @@ async def _complete_workflow_task_with_retry(
15531557
task_id: str,
15541558
attempt: int,
15551559
commands: list[dict[str, Any]],
1560+
message_stream_cursors: list[dict[str, Any]] | None = None,
1561+
message_stream_waits: list[dict[str, Any]] | None = None,
15561562
) -> None:
15571563
for completion_attempt in range(1, _WORKFLOW_TASK_COMPLETION_MAX_ATTEMPTS + 1):
15581564
try:
@@ -1561,6 +1567,8 @@ async def _complete_workflow_task_with_retry(
15611567
lease_owner=self.worker_id,
15621568
workflow_task_attempt=attempt,
15631569
commands=commands,
1570+
message_stream_cursors=message_stream_cursors,
1571+
message_stream_waits=message_stream_waits,
15641572
)
15651573
return
15661574
except Exception as error:

0 commit comments

Comments
 (0)