Skip to content

Commit db0c029

Browse files
authored
Fix stable original query timeout handling (#145)
* Preserve the original canonical query timeout * Prepare compatible query timeout correction 2.5.12 --------- Co-authored-by: Durable Workflow <support@durable-workflow.com>
1 parent 170a4ab commit db0c029

5 files changed

Lines changed: 107 additions & 5 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
66

77
## [Unreleased]
88

9+
## [2.5.12] - 2026-10-13
10+
11+
### Fixed
12+
- Surface the original canonical workflow query timeout instead of automatically
13+
admitting another query through the HTTP transport retry policy.
14+
915
## [2.5.11] - 2026-10-13
1016

1117
### Fixed

‎pyproject.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
44

55
[project]
66
name = "durable-workflow"
7-
version = "2.5.11"
7+
version = "2.5.12"
88
description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
99
readme = "README.md"
1010
requires-python = ">=3.10"
@@ -71,8 +71,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
7171
durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
7272

7373
[tool.durable-workflow]
74-
product-train = "2.5.11"
75-
registry-version = "2.5.11"
74+
product-train = "2.5.12"
75+
registry-version = "2.5.12"
7676
supported-server-versions = "2.5.16"
7777
worker-protocol-version = "1.20"
7878
control-plane-version = "2"

‎src/durable_workflow/client.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4323,7 +4323,9 @@ async def query_workflow(
43234323
Queries are synchronous and non-mutating. The server runs the named
43244324
query handler inside the workflow process and returns the decoded
43254325
result. Raises :class:`~durable_workflow.errors.QueryFailed` if the
4326-
query was rejected or the handler errored.
4326+
query was rejected, the handler errored, or the original query timed
4327+
out. A canonical query timeout is surfaced without automatically
4328+
admitting another query.
43274329
"""
43284330
await self._require_query_support()
43294331
body: dict[str, Any] = {}

‎src/durable_workflow/retry_policy.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import json
1818
import logging
1919
import random
20+
import re
2021
from collections.abc import Awaitable, Callable
2122
from dataclasses import dataclass
2223
from typing import TypeVar
@@ -34,6 +35,24 @@
3435
)
3536

3637

38+
def _query_timeout_refusal(exc: httpx.HTTPStatusError) -> bool:
39+
request = exc.request
40+
if (exc.response.status_code != 504 or request.method != "POST"
41+
or request.headers.get("X-Durable-Workflow-Control-Plane-Version") != "2"
42+
or re.search(r"/api/workflows/[^/]+/query/[^/]+$", request.url.path) is None):
43+
return False
44+
try:
45+
body = exc.response.json()
46+
except ValueError:
47+
return False
48+
# This response ends an admitted query. Another POST creates a new task,
49+
# rather than retrying the original task or its remaining caller budget.
50+
reason = body.get("reason") if isinstance(body, dict) else None
51+
return isinstance(reason, str) and reason in {
52+
"query_worker_timeout", "query_worker_execution_timeout", "query_task_not_claimed",
53+
}
54+
55+
3756
def _storage_refusal(exc: Exception) -> tuple[ServerError, str | None] | None:
3857
if not isinstance(exc, httpx.HTTPStatusError):
3958
return None
@@ -194,6 +213,8 @@ class TransportRetryPolicy:
194213
:class:`durable_workflow.workflow.ChildWorkflowRetryPolicy`.
195214
196215
Uses exponential backoff with jitter to avoid thundering herd.
216+
Canonical query timeouts end the original query and are surfaced without
217+
another query admission; unrelated transient HTTP failures still retry.
197218
"""
198219

199220
max_attempts: int = 3
@@ -213,7 +234,7 @@ def should_retry(self, exc: Exception, attempt: int) -> bool:
213234

214235
# Retry 5xx server errors and 429 rate limit
215236
if isinstance(exc, httpx.HTTPStatusError):
216-
if _poll_capacity_refusal(exc):
237+
if _query_timeout_refusal(exc) or _poll_capacity_refusal(exc):
217238
return False
218239
return exc.response.status_code >= 500 or exc.response.status_code == 429
219240

‎tests/test_query_timeout_retry.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
from __future__ import annotations
2+
3+
import httpx
4+
import pytest
5+
6+
from durable_workflow import Client
7+
from durable_workflow.errors import QueryFailed
8+
from durable_workflow.retry_policy import TransportRetryPolicy
9+
10+
11+
@pytest.mark.parametrize("protocol", ["1.20", "1.19"])
12+
@pytest.mark.parametrize("reason", [
13+
"query_worker_execution_timeout", "query_task_not_claimed", "query_worker_timeout",
14+
])
15+
@pytest.mark.parametrize("prefix", ["", "/api/runtime/v1/namespaces/example"])
16+
async def test_query_exposes_original_canonical_timeout_without_another_admission(
17+
monkeypatch: pytest.MonkeyPatch, protocol: str, reason: str, prefix: str,
18+
) -> None:
19+
monkeypatch.setenv("DURABLE_WORKFLOW_WORKER_PROTOCOL_VERSION", protocol)
20+
original = {"reason": reason, "message": "Original query ended.",
21+
"control_plane": {"operation": "query", "reason": reason}}
22+
admissions = 0
23+
discovery_protocols = []
24+
25+
def handle(request: httpx.Request) -> httpx.Response:
26+
nonlocal admissions
27+
if request.url.path.endswith("/api/cluster/info"):
28+
discovery_protocols.append(request.headers["X-Durable-Workflow-Protocol-Version"])
29+
return httpx.Response(200, json={
30+
"worker_protocol": {"server_capabilities": {"query_tasks": True}},
31+
})
32+
assert request.url.path == prefix + "/api/workflows/original/query/total"
33+
assert request.headers["X-Durable-Workflow-Control-Plane-Version"] == "2"
34+
admissions += 1
35+
if admissions == 1:
36+
return httpx.Response(504, json=original)
37+
# The old policy hides the original failure behind a distinct admission.
38+
return httpx.Response(200, json={"result": "replacement-query"})
39+
40+
async with Client("https://runtime.example" + prefix, worker_token="worker", control_token="control",
41+
retry_policy=TransportRetryPolicy(initial_backoff_seconds=0, jitter=False)) as client:
42+
await client._http.aclose()
43+
client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport(handle))
44+
with pytest.raises(QueryFailed) as failure:
45+
await client.query_workflow("original", "total")
46+
assert admissions == 1
47+
assert discovery_protocols == [protocol]
48+
assert failure.value.status == 504 and failure.value.reason == reason
49+
assert failure.value.body == original
50+
51+
52+
@pytest.mark.parametrize(("method", "path", "status", "body", "headers"), [
53+
("POST", "/api/workflows/w/query/q", 504, {"reason": "gateway_timeout"},
54+
{"X-Durable-Workflow-Control-Plane-Version": "2"}),
55+
("POST", "/api/workflows/w/query/q", 503, {"reason": "query_worker_timeout"},
56+
{"X-Durable-Workflow-Control-Plane-Version": "2"}),
57+
("GET", "/api/workflows/w/query/q", 504, {"reason": "query_worker_timeout"},
58+
{"X-Durable-Workflow-Control-Plane-Version": "2"}),
59+
("POST", "/api/workflows/w/signals/q", 504, {"reason": "query_worker_timeout"},
60+
{"X-Durable-Workflow-Control-Plane-Version": "2"}),
61+
("POST", "/api/workflows/w/query/q", 504, {"reason": "query_worker_timeout"}, {}),
62+
("POST", "/api/workflows/w/query/q", 504, [],
63+
{"X-Durable-Workflow-Control-Plane-Version": "2"}),
64+
("POST", "/api/workflows/w/query/q", 504, {"reason": []},
65+
{"X-Durable-Workflow-Control-Plane-Version": "2"}),
66+
])
67+
def test_other_transport_failures_keep_existing_retry_policy(
68+
method: str, path: str, status: int, body: object, headers: dict[str, str],
69+
) -> None:
70+
request = httpx.Request(method, "https://runtime.example" + path, headers=headers)
71+
response = httpx.Response(status, request=request, json=body)
72+
error = httpx.HTTPStatusError("transport failure", request=request, response=response)
73+
assert TransportRetryPolicy().should_retry(error, 0) is True

0 commit comments

Comments
 (0)