Skip to content

Commit 633e8ae

Browse files
authored
Backport payload lease protection to Python 2.5.9 (#133)
* Renew workflow task leases while hydrating external payloads * Bind payload observers without calling Client during Worker construction * Prepare stable Python 2.5.9 payload lease correction --------- Co-authored-by: Durable Workflow <support@durable-workflow.com>
1 parent b30d38f commit 633e8ae

5 files changed

Lines changed: 276 additions & 7 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
66

77
## [Unreleased]
88

9+
## [2.5.9] - 2026-10-12
10+
11+
### Fixed
12+
- Renew the original workflow task lease while downloading authenticated input
13+
and history payloads. Stop the transfer when authority is lost or the Worker
14+
stops, and preserve cancellation observations for canonical delivery.
15+
916
## [2.5.1] - 2026-10-10
1017

1118
### Fixed

‎pyproject.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
44

55
[project]
66
name = "durable-workflow"
7-
version = "2.5.8"
7+
version = "2.5.9"
88
description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
99
readme = "README.md"
1010
requires-python = ">=3.10"
@@ -71,8 +71,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
7171
durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
7272

7373
[tool.durable-workflow]
74-
product-train = "2.5.8"
75-
registry-version = "2.5.8"
74+
product-train = "2.5.9"
75+
registry-version = "2.5.9"
7676
supported-server-versions = "2.5.16"
7777
worker-protocol-version = "1.20"
7878
control-plane-version = "2"

‎src/durable_workflow/client.py‎

Lines changed: 78 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
import time
2929
import uuid
3030
import warnings
31-
from collections.abc import AsyncIterator, Callable, Mapping, Sequence
31+
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence
3232
from dataclasses import dataclass
3333
from importlib.metadata import PackageNotFoundError
3434
from importlib.metadata import version as _pkg_version
@@ -1624,13 +1624,15 @@ def __init__(
16241624
_RuntimeExternalPayloadTransport | None
16251625
) = None
16261626
self._runtime_external_payload_transport_resolved = False
1627+
self._workflow_payload_observers: dict[str, Callable[[dict[str, Any]], Awaitable[None]]] = {}
16271628

16281629
async def aclose(self) -> None:
16291630
"""Close the underlying ``httpx`` connection pool.
16301631
16311632
Equivalent to exiting the async-context-manager form of the client.
16321633
Safe to call multiple times.
16331634
"""
1635+
self._workflow_payload_observers.clear()
16341636
await self._http.aclose()
16351637

16361638
async def __aenter__(self) -> Client:
@@ -1877,10 +1879,12 @@ async def _do_request() -> httpx.Response:
18771879
"runtime returned an external payload reference without advertising "
18781880
"the authenticated namespace transport"
18791881
)
1880-
result = await self._resolve_runtime_payloads(
1882+
result = await self._resolve_workflow_claim_payloads(
18811883
result,
18821884
worker=worker,
18831885
transport=transport,
1886+
path=path,
1887+
request_json=request_json,
18841888
)
18851889
result = self._hydrate_result_envelopes(result)
18861890
outcome = "ok"
@@ -2251,6 +2255,77 @@ async def _do_request() -> httpx.Response:
22512255
self.external_storage_cache.put(reference, data)
22522256
return reference
22532257

2258+
async def _resolve_workflow_claim_payloads(
2259+
self,
2260+
value: Any,
2261+
*,
2262+
worker: bool,
2263+
transport: _RuntimeExternalPayloadTransport,
2264+
path: str,
2265+
request_json: Any,
2266+
) -> Any:
2267+
lease: dict[str, Any] | None = None
2268+
owner: Any = None
2269+
is_poll = worker and path == "/worker/workflow-tasks/poll"
2270+
history = re.fullmatch(r"/worker/workflow-tasks/([^/]+)/history", path) if worker else None
2271+
if isinstance(request_json, dict) and isinstance(value, dict):
2272+
if is_poll and isinstance(value.get("task"), dict):
2273+
task = value["task"]
2274+
if task.get("task_kind", "workflow") == "workflow":
2275+
lease = dict(task)
2276+
owner = request_json.get("worker_id")
2277+
elif history is not None:
2278+
lease = dict(value)
2279+
lease.update({
2280+
"task_id": unquote(history.group(1)),
2281+
"lease_owner": request_json.get("lease_owner"),
2282+
"workflow_task_attempt": request_json.get("workflow_task_attempt"),
2283+
})
2284+
owner = request_json.get("lease_owner")
2285+
observer = self._workflow_payload_observers.get(owner) if isinstance(owner, str) else None
2286+
if lease is None or observer is None:
2287+
return await self._resolve_runtime_payloads(value, worker=worker, transport=transport)
2288+
if (not isinstance(lease.get("task_id"), str) or not lease["task_id"]
2289+
or lease.get("lease_owner", owner) != owner
2290+
or type(lease.get("workflow_task_attempt", 1)) is not int
2291+
or lease.get("workflow_task_attempt", 1) < 1
2292+
or lease.get("namespace", self.namespace) != self.namespace):
2293+
raise ExternalPayloadUnsupported("workflow payload response has an invalid task ownership fence")
2294+
lease.setdefault("lease_owner", owner)
2295+
lease.setdefault("workflow_task_attempt", 1)
2296+
2297+
async def confirm() -> None:
2298+
await asyncio.wait_for(observer(lease), timeout=5.0)
2299+
2300+
async def observe() -> None:
2301+
while True:
2302+
await asyncio.sleep(5.0)
2303+
await confirm()
2304+
2305+
# The poll body is now available, but application code cannot run until
2306+
# its referenced bytes arrive and the same claim is confirmed again.
2307+
await confirm()
2308+
resolution = asyncio.create_task(self._resolve_runtime_payloads(value, worker=worker, transport=transport))
2309+
observation = asyncio.create_task(observe())
2310+
try:
2311+
done, _ = await asyncio.wait({resolution, observation}, return_when=asyncio.FIRST_COMPLETED)
2312+
if observation in done:
2313+
await observation
2314+
raise ExternalPayloadUnavailable("workflow payload lease observer stopped")
2315+
result = await resolution
2316+
observation.cancel()
2317+
await asyncio.gather(observation, return_exceptions=True)
2318+
await confirm()
2319+
if isinstance(result, dict) and "cancellation_request" in lease:
2320+
target = result.get("task") if is_poll else result
2321+
if isinstance(target, dict):
2322+
target["cancellation_request"] = lease["cancellation_request"]
2323+
return result
2324+
finally:
2325+
resolution.cancel()
2326+
observation.cancel()
2327+
await asyncio.gather(resolution, observation, return_exceptions=True)
2328+
22542329
async def _resolve_runtime_payloads(
22552330
self,
22562331
value: Any,
@@ -2689,6 +2764,7 @@ def _nexus_target_client(self, target_namespace: str | None) -> Client:
26892764
scoped._cluster_info_lock = asyncio.Lock()
26902765
scoped._runtime_external_payload_transport_cache = None
26912766
scoped._runtime_external_payload_transport_resolved = False
2767+
scoped._workflow_payload_observers = {}
26922768
scoped.external_storage = None
26932769
scoped.external_storage_threshold_bytes = None
26942770
scoped.external_storage_cache = ExternalPayloadCache()

‎src/durable_workflow/worker.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1109,6 +1109,9 @@ def __init__(
11091109
self._worker_sessions: dict[str, WorkerSession] = {}
11101110
self._stop = asyncio.Event()
11111111
self._local_activity_shutdown = asyncio.Event()
1112+
payload_observers = getattr(self.client, "_workflow_payload_observers", None)
1113+
if isinstance(payload_observers, dict):
1114+
payload_observers[self.worker_id] = self._renew_workflow_payload_lease
11121115
self._local_activity_executor: ThreadPoolExecutor | None = None
11131116
self._remote_activity_executor: ThreadPoolExecutor | None = None
11141117
self._remote_activity_threads: set[Future[Any]] = set()
@@ -1519,6 +1522,28 @@ async def _resolve_workflow_nexus_commands(
15191522

15201523
raise RuntimeError("workflow yielded too many consecutive Nexus service calls")
15211524

1525+
async def _renew_workflow_payload_lease(self, task: dict[str, Any]) -> None:
1526+
if self._stop.is_set() or self._local_activity_shutdown.is_set():
1527+
raise LocalActivityExecutionAborted("worker shutdown stopped its workflow payload transfer")
1528+
response = await self.client.heartbeat_workflow_task(
1529+
task_id=task["task_id"], lease_owner=self.worker_id,
1530+
workflow_task_attempt=task["workflow_task_attempt"],
1531+
)
1532+
if self._stop.is_set() or self._local_activity_shutdown.is_set():
1533+
raise LocalActivityExecutionAborted("worker shutdown stopped its workflow payload transfer")
1534+
if not isinstance(response, Mapping) or any((
1535+
response.get("task_id") != task["task_id"],
1536+
response.get("lease_owner") != self.worker_id,
1537+
response.get("workflow_task_attempt") != task["workflow_task_attempt"],
1538+
response.get("renewed") is not True,
1539+
)):
1540+
raise LocalActivityExecutionAborted("workflow payload lease was not fenced and acknowledged")
1541+
observation = response.get("cancellation_request")
1542+
if observation is not None:
1543+
# Preserve the observation for the normal canonical delivery path.
1544+
# Hydrating a payload does not deliver or acknowledge cancellation.
1545+
self._observe_workflow_cancellation(task, observation)
1546+
15221547
async def _renew_local_workflow_lease(self, task: dict[str, Any]) -> None:
15231548
if self._cooperative_cancellation_supported and self._local_activity_shutdown.is_set():
15241549
raise LocalActivityExecutionAborted("worker shutdown abandoned its local workflow claim")

‎tests/test_runtime_external_payload_transport.py‎

Lines changed: 163 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
from __future__ import annotations
22

3+
import asyncio
34
import hashlib
45
import json
56
from typing import Any
67

78
import httpx
89
import pytest
910

10-
from durable_workflow import Replayer, serializer, workflow
11+
from durable_workflow import Replayer, Worker, serializer, workflow
1112
from durable_workflow.client import Client
1213
from durable_workflow.errors import (
1314
ExternalPayloadError,
@@ -26,7 +27,7 @@
2627
ExternalPayloadCache,
2728
)
2829
from durable_workflow.retry_policy import TransportRetryPolicy
29-
from durable_workflow.workflow import CompleteWorkflow, WorkflowContext
30+
from durable_workflow.workflow import CompleteWorkflow, LocalActivityExecutionAborted, WorkflowContext
3031

3132

3233
@workflow.defn(name="runtime-payload-replay")
@@ -309,6 +310,166 @@ def runtime_envelope(server: FakeRuntimePayloadServer, value: Any) -> dict[str,
309310
return {"codec": "avro", "external_payload": server.seed(blob)}
310311

311312

313+
@pytest.mark.asyncio
314+
@pytest.mark.parametrize("history_page", [False, True])
315+
async def test_worker_confirms_original_lease_before_and_after_payload_hydration(history_page: bool) -> None:
316+
server = FakeRuntimePayloadServer()
317+
client = runtime_client(server)
318+
Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow])
319+
task = {"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3,
320+
"arguments": runtime_envelope(server, ["original-input"])}
321+
path = "/api/worker/workflow-tasks/task-1/history" if history_page else "/api/worker/workflow-tasks/poll"
322+
server.responses[("POST", path)] = {"history_events": [task]} if history_page else {"task": task}
323+
server.responses[("POST", "/api/worker/workflow-tasks/task-1/heartbeat")] = {
324+
"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, "renewed": True,
325+
}
326+
original_handler = server.handler
327+
328+
def handler(request: httpx.Request) -> httpx.Response:
329+
if request.method == "GET" and "/external-payloads/" in request.url.path:
330+
assert len([entry for entry in server.requests if entry[1].endswith("/heartbeat")]) == 1
331+
return original_handler(request)
332+
333+
client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport(handler))
334+
try:
335+
if history_page:
336+
result = await client.workflow_task_history(task_id="task-1", lease_owner="worker-1",
337+
workflow_task_attempt=3, next_history_page_token="page-2")
338+
resolved = result["history_events"][0]
339+
else:
340+
result = await client.poll_workflow_task(worker_id="worker-1", task_queue="billing", timeout=0)
341+
resolved = result
342+
assert serializer.decode_envelope(resolved["arguments"]) == ["original-input"]
343+
heartbeats = [entry for entry in server.requests if entry[1].endswith("/heartbeat")]
344+
assert len(heartbeats) == 2
345+
assert all(entry[2] == {"lease_owner": "worker-1", "workflow_task_attempt": 3} for entry in heartbeats)
346+
finally:
347+
await client.aclose()
348+
349+
350+
@pytest.mark.asyncio
351+
@pytest.mark.parametrize("changed_fence", [
352+
{"workflow_task_attempt": 4}, {"lease_owner": "worker-2"}, {"renewed": False},
353+
])
354+
async def test_worker_does_not_fetch_payload_when_original_claim_is_not_confirmed(
355+
changed_fence: dict[str, Any],
356+
) -> None:
357+
server = FakeRuntimePayloadServer()
358+
client = runtime_client(server)
359+
Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow])
360+
server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": {
361+
"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3,
362+
"arguments": runtime_envelope(server, ["never-executed"]),
363+
}}
364+
server.responses[("POST", "/api/worker/workflow-tasks/task-1/heartbeat")] = {
365+
"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, "renewed": True, **changed_fence,
366+
}
367+
try:
368+
with pytest.raises(LocalActivityExecutionAborted, match="workflow payload lease was not fenced"):
369+
await client.poll_workflow_task(worker_id="worker-1", task_queue="billing", timeout=0)
370+
assert server.fetch_count == 0
371+
finally:
372+
await client.aclose()
373+
374+
375+
@pytest.mark.asyncio
376+
async def test_payload_renewal_preserves_new_cancellation_for_canonical_delivery() -> None:
377+
from tests.test_cooperative_cancellation import observation
378+
379+
server = FakeRuntimePayloadServer()
380+
client = runtime_client(server)
381+
worker = Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow],
382+
capabilities=["cooperative_cancellation"])
383+
worker._cooperative_cancellation_supported = True
384+
original = observation()
385+
server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": {
386+
"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3,
387+
"arguments": runtime_envelope(server, ["original-input"]),
388+
}}
389+
server.responses[("POST", "/api/worker/workflow-tasks/task-1/heartbeat")] = {
390+
"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, "renewed": True,
391+
"cancellation_request": original,
392+
}
393+
try:
394+
result = await client.poll_workflow_task(worker_id="worker-1", task_queue="billing", timeout=0)
395+
assert result["cancellation_request"] == original
396+
assert "_delivered_cancellation_request_id" not in result
397+
assert not any("/cancellation/" in entry[1] for entry in server.requests)
398+
finally:
399+
await client.aclose()
400+
401+
402+
@pytest.mark.asyncio
403+
async def test_stopped_worker_does_not_start_claim_payload_download() -> None:
404+
server = FakeRuntimePayloadServer()
405+
client = runtime_client(server)
406+
worker = Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow])
407+
worker._stop.set()
408+
server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": {
409+
"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3,
410+
"arguments": runtime_envelope(server, ["never-executed"]),
411+
}}
412+
try:
413+
with pytest.raises(LocalActivityExecutionAborted, match="shutdown stopped"):
414+
await client.poll_workflow_task(worker_id="worker-1", task_queue="billing", timeout=0)
415+
assert server.fetch_count == 0
416+
assert not any(entry[1].endswith("/heartbeat") for entry in server.requests)
417+
finally:
418+
await client.aclose()
419+
420+
421+
@pytest.mark.asyncio
422+
async def test_payload_transfer_is_cancelled_when_its_periodic_lease_observer_loses_authority(
423+
monkeypatch: pytest.MonkeyPatch,
424+
) -> None:
425+
server = FakeRuntimePayloadServer()
426+
client = runtime_client(server)
427+
Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow])
428+
server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": {
429+
"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3,
430+
"arguments": runtime_envelope(server, ["never-executed"]),
431+
}}
432+
fetch_started = asyncio.Event()
433+
fetch_cancelled = asyncio.Event()
434+
observations = 0
435+
436+
async def heartbeat(request: httpx.Request) -> httpx.Response:
437+
nonlocal observations
438+
if request.url.path.endswith("/heartbeat"):
439+
observations += 1
440+
if observations > 1:
441+
return httpx.Response(409, json={"reason": "lease_expired"})
442+
return httpx.Response(200, json={"task_id": "task-1", "lease_owner": "worker-1",
443+
"workflow_task_attempt": 3, "renewed": True})
444+
if request.method == "GET" and "/external-payloads/" in request.url.path:
445+
fetch_started.set()
446+
try:
447+
await asyncio.Event().wait()
448+
finally:
449+
fetch_cancelled.set()
450+
return server.handler(request)
451+
452+
real_sleep = asyncio.sleep
453+
454+
async def prompt_observation(delay: float) -> None:
455+
if delay == 5.0:
456+
await fetch_started.wait()
457+
await real_sleep(0)
458+
else:
459+
await real_sleep(delay)
460+
461+
monkeypatch.setattr("durable_workflow.client.asyncio.sleep", prompt_observation)
462+
client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport(heartbeat))
463+
try:
464+
with pytest.raises(ServerError) as failure:
465+
await asyncio.wait_for(client.poll_workflow_task(worker_id="worker-1", task_queue="billing", timeout=0), 2)
466+
assert failure.value.reason() == "lease_expired"
467+
assert fetch_cancelled.is_set()
468+
assert observations == 2
469+
finally:
470+
await client.aclose()
471+
472+
312473
@pytest.mark.asyncio
313474
@pytest.mark.parametrize("threshold_bytes", [1, 4096])
314475
async def test_nexus_arguments_offload_without_changing_recorded_arguments(threshold_bytes: int) -> None:

0 commit comments

Comments
 (0)