Skip to content

Commit 43be84a

Browse files
authored
Merge pull request #220 from durable-workflow/fix/526-poll-progress
Bound worker poll progress by the HTTP attempt deadline
2 parents 3511924 + e18020a commit 43be84a

7 files changed

Lines changed: 368 additions & 8 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ jobs:
200200
integration:
201201
runs-on: ubuntu-latest
202202
timeout-minutes: 25
203-
needs: [lint, test]
203+
needs: [lint]
204204
env:
205205
DURABLE_WORKFLOW_COOPERATIVE_QUALIFICATION: ${{ inputs.cooperative_qualification && '1' || '0' }}
206206
DURABLE_WORKFLOW_NATIVE_SOURCE: ${{ github.workspace }}/native

‎CHANGELOG.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
66

77
## [Unreleased]
88

9+
## [2.11.2] - 2026-10-25
10+
11+
### Fixed
12+
13+
- Bound each worker poll HTTP attempt with an absolute deadline, including
14+
responses that keep sending progress. Preserve cancellation, retry budgets
15+
and the original poll identity.
16+
- Accept negotiated poll-progress responses alongside ordinary JSON, applying
17+
their original status and retry headers before normal error handling.
18+
- Warn when a connection failure or gateway error cuts a poll short, naming
19+
the observed duration and configured wait.
20+
921
## [2.11.1] - 2026-10-24
1022

1123
### Fixed

‎docs/reference/worker.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,20 @@ the application type would change which failure the runtime records.
7373

7474
## Poll failures and recovery
7575

76+
Worker poll requests advertise optional JSON progress responses and also accept
77+
the ordinary JSON response from Servers that do not implement that transport.
78+
When a Server sends progress while waiting, each HTTP attempt still has an
79+
absolute deadline. Receiving another progress byte does not extend it. The
80+
existing transport retry budget and logical poll identity continue to apply.
81+
The SDK interprets the terminal status and retry headers before the ordinary
82+
error handling, including errors that arrive after HTTP 200 progress began.
83+
84+
If a connection failure, HTTP 502 or HTTP 504 ends a poll before its requested
85+
wait, a warning names the observed duration and configured wait. Check the
86+
proxy and HTTP timeout budgets. An intentional cancellation does not produce
87+
this cutoff warning. These transport observations do not establish that a task
88+
was delivered or authorize revoking its lease.
89+
7690
This backoff requires Python SDK 2.10.0 or later. Earlier packages use a fixed
7791
one-second poll-error wait.
7892

‎pyproject.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
44

55
[project]
66
name = "durable-workflow"
7-
version = "2.11.1"
7+
version = "2.11.2"
88
description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
99
readme = "README.md"
1010
requires-python = ">=3.10"
@@ -76,8 +76,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
7676
durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
7777

7878
[tool.durable-workflow]
79-
product-train = "2.11.1"
80-
registry-version = "2.11.1"
79+
product-train = "2.11.2"
80+
registry-version = "2.11.2"
8181
supported-server-versions = "2.8.0"
8282
worker-protocol-version = "1.20"
8383
control-plane-version = "2"
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
"""Negotiated long-poll responses and early transport failure diagnostics."""
2+
3+
from __future__ import annotations
4+
5+
import logging
6+
import time
7+
from collections.abc import Mapping
8+
from typing import Any
9+
10+
import httpx
11+
12+
MEDIA_TYPE = "application/vnd.durable-workflow.poll.v1+json"
13+
ACCEPT = f"{MEDIA_TYPE}, application/json;q=0.9"
14+
_SCHEMA = "durable-workflow.poll-transport.v1"
15+
POLL_PATHS = frozenset(
16+
f"/worker/{kind}-tasks/poll" for kind in ("workflow", "activity", "query", "update-validation")
17+
)
18+
_HEADERS = frozenset(("retry-after", "x-durable-workflow-protocol-version"))
19+
log = logging.getLogger(__name__)
20+
21+
22+
def unwrap_response(response: httpx.Response) -> httpx.Response:
23+
if response.headers.get("content-type", "").split(";", 1)[0].strip().lower() != MEDIA_TYPE:
24+
return response
25+
26+
try:
27+
envelope = response.json()
28+
if not isinstance(envelope, dict) or envelope.get("schema") != _SCHEMA:
29+
raise ValueError("unrecognized poll transport")
30+
status, headers, body = envelope.get("status"), envelope.get("headers"), envelope.get("body")
31+
if (type(status) is not int or not 200 <= status < 600
32+
or response.status_code not in (200, status)
33+
or not isinstance(body, dict) or not isinstance(headers, dict)
34+
or any(key not in _HEADERS or not isinstance(value, str)
35+
or "\r" in value or "\n" in value for key, value in headers.items())):
36+
raise ValueError("invalid poll transport result")
37+
# Apply the original status and retry headers before the normal client
38+
# error handling. Outer HTTP 200 may have been committed by progress.
39+
return httpx.Response(status, headers=headers, json=body, request=response.request)
40+
except (ValueError, TypeError) as error:
41+
raise httpx.RemoteProtocolError("Invalid worker poll transport response.", request=response.request) from error
42+
43+
44+
def warn_early_cutoff(path: str, body: Any, started: float, error: Exception) -> None:
45+
wait = body.get("timeout_seconds") if isinstance(body, Mapping) else None
46+
observed = time.monotonic() - started
47+
if isinstance(wait, bool) or not isinstance(wait, int | float) or wait <= 0 or observed >= wait:
48+
return
49+
if isinstance(error, httpx.HTTPStatusError) and error.response.status_code not in (502, 504):
50+
return
51+
log.warning(
52+
"Worker long poll %s ended after %.3fs before its configured wait of %.3fs (%s). "
53+
"Check proxy and HTTP timeout budgets.",
54+
path, observed, wait, type(error).__name__,
55+
)

‎src/durable_workflow/client.py‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@
5757
prepare_failure_request,
5858
)
5959
from ._metric_routes import route_for_metrics as _route_for_metrics
60+
from ._poll_transport import ACCEPT as _POLL_ACCEPT
61+
from ._poll_transport import POLL_PATHS as _POLL_PATHS
62+
from ._poll_transport import unwrap_response as _unwrap_poll_response
63+
from ._poll_transport import warn_early_cutoff as _warn_poll_cutoff
6064
from ._query_task_authority import QueryTaskAuthority, current_query_authority
6165
from .errors import (
6266
REQUEST_ID_CONFLICT,
@@ -1882,6 +1886,7 @@ async def _request(
18821886

18831887
query_poll = worker and path == "/worker/query-tasks/poll"
18841888
activity_poll = worker and path == "/worker/activity-tasks/poll"
1889+
progress_poll = worker and method.upper() == "POST" and path in _POLL_PATHS
18851890
if query_poll:
18861891
current_query_authority.set(None)
18871892
if activity_poll:
@@ -1931,15 +1936,34 @@ async def _do_request() -> httpx.Response:
19311936
request_timeout = self.timeout if timeout is None else timeout
19321937
if worker and activity_authority is not None and activity_authority.governs(self, self.namespace, path):
19331938
request_timeout = min(request_timeout, activity_authority.remaining())
1934-
resp = await self._http.request(
1939+
headers = self._discovery_headers() if discovery else self._headers(worker=worker)
1940+
if progress_poll:
1941+
headers["Accept"] = _POLL_ACCEPT
1942+
pending = self._http.request(
19351943
method,
19361944
f"/api{path}",
1937-
headers=self._discovery_headers() if discovery else self._headers(worker=worker),
1945+
headers=headers,
19381946
json=request_json,
19391947
timeout=request_timeout,
19401948
)
1941-
# Raise HTTPStatusError for 4xx/5xx so retry policy can catch it
1942-
resp.raise_for_status()
1949+
try:
1950+
if progress_poll:
1951+
try:
1952+
resp = await asyncio.wait_for(pending, timeout=request_timeout)
1953+
except asyncio.TimeoutError as error:
1954+
raise httpx.ReadTimeout(
1955+
"Worker poll exceeded its absolute HTTP attempt deadline.",
1956+
request=self._http.build_request(method, f"/api{path}"),
1957+
) from error
1958+
resp = _unwrap_poll_response(resp)
1959+
else:
1960+
resp = await pending
1961+
# Interpret negotiated late errors before the usual retry policy.
1962+
resp.raise_for_status()
1963+
except (httpx.TransportError, httpx.HTTPStatusError) as error:
1964+
if progress_poll:
1965+
_warn_poll_cutoff(path, request_json, request_started, error)
1966+
raise
19431967
return resp
19441968

19451969
try:

0 commit comments

Comments
 (0)