|
1 | 1 | from __future__ import annotations |
2 | 2 |
|
| 3 | +import asyncio |
3 | 4 | import hashlib |
4 | 5 | import json |
5 | 6 | from typing import Any |
6 | 7 |
|
7 | 8 | import httpx |
8 | 9 | import pytest |
9 | 10 |
|
10 | | -from durable_workflow import Replayer, serializer, workflow |
| 11 | +from durable_workflow import Replayer, Worker, serializer, workflow |
11 | 12 | from durable_workflow.client import Client |
12 | 13 | from durable_workflow.errors import ( |
13 | 14 | ExternalPayloadError, |
|
26 | 27 | ExternalPayloadCache, |
27 | 28 | ) |
28 | 29 | from durable_workflow.retry_policy import TransportRetryPolicy |
29 | | -from durable_workflow.workflow import CompleteWorkflow, WorkflowContext |
| 30 | +from durable_workflow.workflow import CompleteWorkflow, LocalActivityExecutionAborted, WorkflowContext |
30 | 31 |
|
31 | 32 |
|
32 | 33 | @workflow.defn(name="runtime-payload-replay") |
@@ -309,6 +310,166 @@ def runtime_envelope(server: FakeRuntimePayloadServer, value: Any) -> dict[str, |
309 | 310 | return {"codec": "avro", "external_payload": server.seed(blob)} |
310 | 311 |
|
311 | 312 |
|
| 313 | +@pytest.mark.asyncio |
| 314 | +@pytest.mark.parametrize("history_page", [False, True]) |
| 315 | +async def test_worker_confirms_original_lease_before_and_after_payload_hydration(history_page: bool) -> None: |
| 316 | + server = FakeRuntimePayloadServer() |
| 317 | + client = runtime_client(server) |
| 318 | + Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow]) |
| 319 | + task = {"task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, |
| 320 | + "arguments": runtime_envelope(server, ["original-input"])} |
| 321 | + path = "/api/worker/workflow-tasks/task-1/history" if history_page else "/api/worker/workflow-tasks/poll" |
| 322 | + server.responses[("POST", path)] = {"history_events": [task]} if history_page else {"task": task} |
| 323 | + server.responses[("POST", "/api/worker/workflow-tasks/task-1/heartbeat")] = { |
| 324 | + "task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, "renewed": True, |
| 325 | + } |
| 326 | + original_handler = server.handler |
| 327 | + |
| 328 | + def handler(request: httpx.Request) -> httpx.Response: |
| 329 | + if request.method == "GET" and "/external-payloads/" in request.url.path: |
| 330 | + assert len([entry for entry in server.requests if entry[1].endswith("/heartbeat")]) == 1 |
| 331 | + return original_handler(request) |
| 332 | + |
| 333 | + client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport(handler)) |
| 334 | + try: |
| 335 | + if history_page: |
| 336 | + result = await client.workflow_task_history(task_id="task-1", lease_owner="worker-1", |
| 337 | + workflow_task_attempt=3, next_history_page_token="page-2") |
| 338 | + resolved = result["history_events"][0] |
| 339 | + else: |
| 340 | + result = await client.poll_workflow_task(worker_id="worker-1", task_queue="billing", timeout=0) |
| 341 | + resolved = result |
| 342 | + assert serializer.decode_envelope(resolved["arguments"]) == ["original-input"] |
| 343 | + heartbeats = [entry for entry in server.requests if entry[1].endswith("/heartbeat")] |
| 344 | + assert len(heartbeats) == 2 |
| 345 | + assert all(entry[2] == {"lease_owner": "worker-1", "workflow_task_attempt": 3} for entry in heartbeats) |
| 346 | + finally: |
| 347 | + await client.aclose() |
| 348 | + |
| 349 | + |
| 350 | +@pytest.mark.asyncio |
| 351 | +@pytest.mark.parametrize("changed_fence", [ |
| 352 | + {"workflow_task_attempt": 4}, {"lease_owner": "worker-2"}, {"renewed": False}, |
| 353 | +]) |
| 354 | +async def test_worker_does_not_fetch_payload_when_original_claim_is_not_confirmed( |
| 355 | + changed_fence: dict[str, Any], |
| 356 | +) -> None: |
| 357 | + server = FakeRuntimePayloadServer() |
| 358 | + client = runtime_client(server) |
| 359 | + Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow]) |
| 360 | + server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": { |
| 361 | + "task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, |
| 362 | + "arguments": runtime_envelope(server, ["never-executed"]), |
| 363 | + }} |
| 364 | + server.responses[("POST", "/api/worker/workflow-tasks/task-1/heartbeat")] = { |
| 365 | + "task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, "renewed": True, **changed_fence, |
| 366 | + } |
| 367 | + try: |
| 368 | + with pytest.raises(LocalActivityExecutionAborted, match="workflow payload lease was not fenced"): |
| 369 | + await client.poll_workflow_task(worker_id="worker-1", task_queue="billing", timeout=0) |
| 370 | + assert server.fetch_count == 0 |
| 371 | + finally: |
| 372 | + await client.aclose() |
| 373 | + |
| 374 | + |
| 375 | +@pytest.mark.asyncio |
| 376 | +async def test_payload_renewal_preserves_new_cancellation_for_canonical_delivery() -> None: |
| 377 | + from tests.test_cooperative_cancellation import observation |
| 378 | + |
| 379 | + server = FakeRuntimePayloadServer() |
| 380 | + client = runtime_client(server) |
| 381 | + worker = Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow], |
| 382 | + capabilities=["cooperative_cancellation"]) |
| 383 | + worker._cooperative_cancellation_supported = True |
| 384 | + original = observation() |
| 385 | + server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": { |
| 386 | + "task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, |
| 387 | + "arguments": runtime_envelope(server, ["original-input"]), |
| 388 | + }} |
| 389 | + server.responses[("POST", "/api/worker/workflow-tasks/task-1/heartbeat")] = { |
| 390 | + "task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, "renewed": True, |
| 391 | + "cancellation_request": original, |
| 392 | + } |
| 393 | + try: |
| 394 | + result = await client.poll_workflow_task(worker_id="worker-1", task_queue="billing", timeout=0) |
| 395 | + assert result["cancellation_request"] == original |
| 396 | + assert "_delivered_cancellation_request_id" not in result |
| 397 | + assert not any("/cancellation/" in entry[1] for entry in server.requests) |
| 398 | + finally: |
| 399 | + await client.aclose() |
| 400 | + |
| 401 | + |
| 402 | +@pytest.mark.asyncio |
| 403 | +async def test_stopped_worker_does_not_start_claim_payload_download() -> None: |
| 404 | + server = FakeRuntimePayloadServer() |
| 405 | + client = runtime_client(server) |
| 406 | + worker = Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow]) |
| 407 | + worker._stop.set() |
| 408 | + server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": { |
| 409 | + "task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, |
| 410 | + "arguments": runtime_envelope(server, ["never-executed"]), |
| 411 | + }} |
| 412 | + try: |
| 413 | + with pytest.raises(LocalActivityExecutionAborted, match="shutdown stopped"): |
| 414 | + await client.poll_workflow_task(worker_id="worker-1", task_queue="billing", timeout=0) |
| 415 | + assert server.fetch_count == 0 |
| 416 | + assert not any(entry[1].endswith("/heartbeat") for entry in server.requests) |
| 417 | + finally: |
| 418 | + await client.aclose() |
| 419 | + |
| 420 | + |
| 421 | +@pytest.mark.asyncio |
| 422 | +async def test_payload_transfer_is_cancelled_when_its_periodic_lease_observer_loses_authority( |
| 423 | + monkeypatch: pytest.MonkeyPatch, |
| 424 | +) -> None: |
| 425 | + server = FakeRuntimePayloadServer() |
| 426 | + client = runtime_client(server) |
| 427 | + Worker(client, task_queue="billing", worker_id="worker-1", workflows=[RuntimePayloadReplayWorkflow]) |
| 428 | + server.responses[("POST", "/api/worker/workflow-tasks/poll")] = {"task": { |
| 429 | + "task_id": "task-1", "lease_owner": "worker-1", "workflow_task_attempt": 3, |
| 430 | + "arguments": runtime_envelope(server, ["never-executed"]), |
| 431 | + }} |
| 432 | + fetch_started = asyncio.Event() |
| 433 | + fetch_cancelled = asyncio.Event() |
| 434 | + observations = 0 |
| 435 | + |
| 436 | + async def heartbeat(request: httpx.Request) -> httpx.Response: |
| 437 | + nonlocal observations |
| 438 | + if request.url.path.endswith("/heartbeat"): |
| 439 | + observations += 1 |
| 440 | + if observations > 1: |
| 441 | + return httpx.Response(409, json={"reason": "lease_expired"}) |
| 442 | + return httpx.Response(200, json={"task_id": "task-1", "lease_owner": "worker-1", |
| 443 | + "workflow_task_attempt": 3, "renewed": True}) |
| 444 | + if request.method == "GET" and "/external-payloads/" in request.url.path: |
| 445 | + fetch_started.set() |
| 446 | + try: |
| 447 | + await asyncio.Event().wait() |
| 448 | + finally: |
| 449 | + fetch_cancelled.set() |
| 450 | + return server.handler(request) |
| 451 | + |
| 452 | + real_sleep = asyncio.sleep |
| 453 | + |
| 454 | + async def prompt_observation(delay: float) -> None: |
| 455 | + if delay == 5.0: |
| 456 | + await fetch_started.wait() |
| 457 | + await real_sleep(0) |
| 458 | + else: |
| 459 | + await real_sleep(delay) |
| 460 | + |
| 461 | + monkeypatch.setattr("durable_workflow.client.asyncio.sleep", prompt_observation) |
| 462 | + client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport(heartbeat)) |
| 463 | + try: |
| 464 | + with pytest.raises(ServerError) as failure: |
| 465 | + await asyncio.wait_for(client.poll_workflow_task(worker_id="worker-1", task_queue="billing", timeout=0), 2) |
| 466 | + assert failure.value.reason() == "lease_expired" |
| 467 | + assert fetch_cancelled.is_set() |
| 468 | + assert observations == 2 |
| 469 | + finally: |
| 470 | + await client.aclose() |
| 471 | + |
| 472 | + |
312 | 473 | @pytest.mark.asyncio |
313 | 474 | @pytest.mark.parametrize("threshold_bytes", [1, 4096]) |
314 | 475 | async def test_nexus_arguments_offload_without_changing_recorded_arguments(threshold_bytes: int) -> None: |
|
0 commit comments