Skip to content

Commit dfee119

Browse files
authored
Finalize stable 2.5.9 Worker observer cleanup (#135)
* Release payload observer references after Worker shutdown * Wait for the observed heartbeat recovery in its regression test * Use a built-in type for the heartbeat test callback --------- Co-authored-by: Durable Workflow <support@durable-workflow.com>
1 parent 633e8ae commit dfee119

3 files changed

Lines changed: 73 additions & 17 deletions

File tree

‎src/durable_workflow/worker.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1109,9 +1109,6 @@ def __init__(
11091109
self._worker_sessions: dict[str, WorkerSession] = {}
11101110
self._stop = asyncio.Event()
11111111
self._local_activity_shutdown = asyncio.Event()
1112-
payload_observers = getattr(self.client, "_workflow_payload_observers", None)
1113-
if isinstance(payload_observers, dict):
1114-
payload_observers[self.worker_id] = self._renew_workflow_payload_lease
11151112
self._local_activity_executor: ThreadPoolExecutor | None = None
11161113
self._remote_activity_executor: ThreadPoolExecutor | None = None
11171114
self._remote_activity_threads: set[Future[Any]] = set()
@@ -4439,6 +4436,9 @@ def _begin_run(self) -> None:
44394436
if self._stop.is_set():
44404437
raise RuntimeError("worker cannot run after stop() has been called")
44414438
self._run_started = True
4439+
payload_observers = getattr(self.client, "_workflow_payload_observers", None)
4440+
if isinstance(payload_observers, dict):
4441+
payload_observers[self.worker_id] = self._renew_workflow_payload_lease
44424442

44434443
def worker_session(self, options: WorkerSessionOptions) -> WorkerSession:
44444444
"""Return a typed lease handle held by this worker until shutdown."""
@@ -4557,6 +4557,10 @@ async def _shutdown(self) -> None:
45574557
self._registered = False
45584558
log.info("worker %s deregistered", self.worker_id)
45594559
self._sticky_cache.clear()
4560+
payload_observers = getattr(self.client, "_workflow_payload_observers", None)
4561+
if (isinstance(payload_observers, dict)
4562+
and payload_observers.get(self.worker_id) == self._renew_workflow_payload_lease):
4563+
payload_observers.pop(self.worker_id)
45604564

45614565
@staticmethod
45624566
def _remaining_shutdown_time(deadline: float) -> float:

‎tests/test_runtime_external_payload_transport.py‎

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
from __future__ import annotations
22

33
import asyncio
4+
import gc
45
import hashlib
56
import json
7+
import weakref
68
from typing import Any
79

810
import httpx
@@ -310,12 +312,20 @@ def runtime_envelope(server: FakeRuntimePayloadServer, value: Any) -> dict[str,
310312
return {"codec": "avro", "external_payload": server.seed(blob)}
311313

312314

315+
def payload_worker(client: Client, *, capabilities: list[str] | None = None) -> Worker:
316+
worker = Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow],
317+
capabilities=capabilities or [])
318+
worker._begin_run()
319+
worker._registration_done.set()
320+
return worker
321+
322+
313323
@pytest.mark.asyncio
314324
@pytest.mark.parametrize("history_page", [False, True])
315325
async def test_worker_confirms_original_lease_before_and_after_payload_hydration(history_page: bool) -> None:
316326
server = FakeRuntimePayloadServer()
317327
client = runtime_client(server)
318-
Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow])
328+
payload_worker(client)
319329
task = {"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3,
320330
"arguments": runtime_envelope(server, ["original-input"])}
321331
path = "/api/worker/workflow-tasks/task-1/history" if history_page else "/api/worker/workflow-tasks/poll"
@@ -356,7 +366,7 @@ async def test_worker_does_not_fetch_payload_when_original_claim_is_not_confirme
356366
) -> None:
357367
server = FakeRuntimePayloadServer()
358368
client = runtime_client(server)
359-
Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow])
369+
payload_worker(client)
360370
server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": {
361371
"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3,
362372
"arguments": runtime_envelope(server, ["never-executed"]),
@@ -378,8 +388,7 @@ async def test_payload_renewal_preserves_new_cancellation_for_canonical_delivery
378388

379389
server = FakeRuntimePayloadServer()
380390
client = runtime_client(server)
381-
worker = Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow],
382-
capabilities=["cooperative_cancellation"])
391+
worker = payload_worker(client, capabilities=["cooperative_cancellation"])
383392
worker._cooperative_cancellation_supported = True
384393
original = observation()
385394
server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": {
@@ -403,7 +412,7 @@ async def test_payload_renewal_preserves_new_cancellation_for_canonical_delivery
403412
async def test_stopped_worker_does_not_start_claim_payload_download() -> None:
404413
server = FakeRuntimePayloadServer()
405414
client = runtime_client(server)
406-
worker = Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow])
415+
worker = payload_worker(client)
407416
worker._stop.set()
408417
server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": {
409418
"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3,
@@ -424,7 +433,9 @@ async def test_payload_transfer_is_cancelled_when_its_periodic_lease_observer_lo
424433
) -> None:
425434
server = FakeRuntimePayloadServer()
426435
client = runtime_client(server)
427-
Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow])
436+
payload_worker(client)
437+
438+
428439
server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": {
429440
"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3,
430441
"arguments": runtime_envelope(server, ["never-executed"]),
@@ -470,6 +481,36 @@ async def prompt_observation(delay: float) -> None:
470481
await client.aclose()
471482

472483

484+
@pytest.mark.asyncio
485+
async def test_shared_client_does_not_retain_unstarted_or_stopped_workers() -> None:
486+
client = runtime_client(FakeRuntimePayloadServer())
487+
worker = Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow])
488+
assert client._workflow_payload_observers == {}
489+
reference = weakref.ref(worker)
490+
worker._begin_run()
491+
worker._registration_done.set()
492+
assert "worker-1" in client._workflow_payload_observers
493+
await worker.stop()
494+
assert client._workflow_payload_observers == {}
495+
del worker
496+
await asyncio.sleep(0)
497+
gc.collect()
498+
assert reference() is None
499+
await client.aclose()
500+
501+
502+
@pytest.mark.asyncio
503+
async def test_stopping_old_worker_preserves_a_new_worker_observer() -> None:
504+
client = runtime_client(FakeRuntimePayloadServer())
505+
older = payload_worker(client)
506+
newer = payload_worker(client)
507+
await older.stop()
508+
assert client._workflow_payload_observers["worker-1"] == newer._renew_workflow_payload_lease
509+
await newer.stop()
510+
assert client._workflow_payload_observers == {}
511+
await client.aclose()
512+
513+
473514
@pytest.mark.asyncio
474515
@pytest.mark.parametrize("threshold_bytes", [1, 4096])
475516
async def test_nexus_arguments_offload_without_changing_recorded_arguments(threshold_bytes: int) -> None:

‎tests/test_worker.py‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4955,9 +4955,18 @@ async def test_register_adopts_server_advertised_heartbeat_cadence(
49554955
async def test_heartbeat_loop_survives_transient_errors(
49564956
self, mock_client: AsyncMock
49574957
) -> None:
4958-
mock_client.heartbeat_worker = AsyncMock(
4959-
side_effect=[RuntimeError("temporary"), {"acknowledged": True}]
4960-
)
4958+
recovered = asyncio.Event()
4959+
attempts = 0
4960+
4961+
async def heartbeat(**kwargs: object) -> dict[str, bool]:
4962+
nonlocal attempts
4963+
attempts += 1
4964+
if attempts == 1:
4965+
raise RuntimeError("temporary")
4966+
recovered.set()
4967+
return {"acknowledged": True}
4968+
4969+
mock_client.heartbeat_worker = AsyncMock(side_effect=heartbeat)
49614970
worker = Worker(
49624971
mock_client,
49634972
task_queue="q1",
@@ -4967,11 +4976,13 @@ async def test_heartbeat_loop_survives_transient_errors(
49674976
heartbeat_interval=0.02,
49684977
)
49694978
run_task = asyncio.create_task(worker.run())
4970-
await asyncio.sleep(0.15)
4971-
await worker.stop()
4972-
run_task.cancel()
4973-
with contextlib.suppress(asyncio.CancelledError):
4974-
await run_task
4979+
try:
4980+
await asyncio.wait_for(recovered.wait(), timeout=2)
4981+
finally:
4982+
await worker.stop()
4983+
run_task.cancel()
4984+
with contextlib.suppress(asyncio.CancelledError):
4985+
await run_task
49754986
assert mock_client.heartbeat_worker.call_count >= 2
49764987

49774988
def test_process_metrics_cpu_percent_is_instantaneous_not_lifetime(

0 commit comments

Comments
 (0)