Skip to content

Commit 993419e

Browse files
authored
Keep authored Nexus completion waits on durable history (#117)
* Keep Nexus completion waits on durable workflow history * Preserve committed service failure cause when snapshot fields are null --------- Co-authored-by: Durable Workflow <support@durable-workflow.com>
1 parent a3d5372 commit 993419e

11 files changed

Lines changed: 377 additions & 208 deletions

‎docs/reference/client.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
`Client.execute_nexus_operation(...)` calls a registered Nexus service
44
operation through the service-catalog API and returns `NexusOperationResult`.
55
Workflow authors should normally yield `ctx.call_nexus_service(...)`; the
6-
worker uses this client method to execute once and record the durable result
7-
or typed service failure.
6+
worker submits a leased service command and resumes from durable service
7+
history. The client method is available for calls outside workflow execution.
88

99
::: durable_workflow.client

‎docs/reference/workflow.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ Workflow retry and timeout settings are durable command budgets. Use
66
attempts. Use `TransportRetryPolicy` only for client HTTP retries.
77

88
Use `ctx.call_nexus_service(...)` from workflow code to start a Nexus service
9-
operation durably. The Python worker records the accepted response or typed
10-
service failure as a side-effect marker, then replay resumes with
11-
`NexusOperationResult` or raises `NexusOperationFailed` at the yield point.
9+
operation durably. Admission uses the leased workflow command. Synchronous
10+
completion waits remain suspended until a durable terminal service event,
11+
then replay returns `NexusOperationResult` or raises `NexusOperationFailed`
12+
at the yield point. Use `ctx.start_nexus_operation(...)` to return after
13+
acceptance. Older recorded Nexus side-effect results still replay.
1214

1315
Yield a list for deterministic parallel composition. Lists may nest and may
1416
mix activity, child-workflow, and timer commands; the resolved value has the

‎docs/sdk-reference.md‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -378,10 +378,12 @@ registration order as structured fields.
378378
## Nexus service calls
379379

380380
Workflow code can call a registered Nexus service operation through
381-
`WorkflowContext.call_nexus_service(...)`. The worker executes the service
382-
operation through the service-catalog API, records the response or typed
383-
failure as a durable side-effect marker, and resumes replay from that marker
384-
on subsequent workflow tasks.
381+
`WorkflowContext.call_nexus_service(...)`. The worker submits admission with
382+
its leased workflow task. A synchronous completion wait stays suspended until
383+
the Server records `ServiceCallCompleted`, `ServiceCallFailed` or
384+
`ServiceCallCancelled`. Replacement workers replay the same durable result.
385+
`start_nexus_operation(...)` returns after acceptance. Existing histories with
386+
recorded Nexus side-effect results remain readable.
385387

386388
```python
387389
from durable_workflow import NexusOperationFailed

‎src/durable_workflow/client.py‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -300,8 +300,8 @@ def _contains_inline_payload_envelope(value: object) -> bool:
300300
if set(value) == {"codec", "blob"} and isinstance(value.get("blob"), str):
301301
return True
302302
if (
303-
value.get("type") == "record_side_effect"
304-
and isinstance(value.get("result"), str)
303+
(value.get("type") == "record_side_effect" and isinstance(value.get("result"), str))
304+
or (value.get("type") == "start_service_operation" and isinstance(value.get("request_payload"), str))
305305
):
306306
return True
307307
return any(_contains_inline_payload_envelope(item) for item in value.values())
@@ -2080,21 +2080,23 @@ async def _externalize_runtime_payloads(
20802080
slot: tuple[str | int, ...] = (),
20812081
) -> Any:
20822082
if isinstance(value, dict):
2083-
if (
2084-
value.get("type") == "record_side_effect"
2085-
and isinstance(value.get("result"), str)
2086-
):
2083+
command_type = value.get("type")
2084+
blob_key = (
2085+
{"record_side_effect": "result", "start_service_operation": "request_payload"}.get(command_type)
2086+
if isinstance(command_type, str) else None
2087+
)
2088+
if blob_key is not None and isinstance(value.get(blob_key), str):
20872089
normalized_command = dict(value)
20882090
externalized_result = await self._externalize_runtime_payloads(
2089-
{"codec": serializer.AVRO_CODEC, "blob": value["result"]},
2091+
{"codec": value.get("payload_codec", serializer.AVRO_CODEC), "blob": value[blob_key]},
20902092
worker=worker,
20912093
transport=transport,
20922094
uploaded=uploaded,
20932095
completion=completion,
2094-
slot=(*slot, "result"),
2096+
slot=(*slot, blob_key),
20952097
)
20962098
if "external_payload" in externalized_result:
2097-
normalized_command["result"] = externalized_result
2099+
normalized_command[blob_key] = externalized_result
20982100
value = normalized_command
20992101

21002102
if set(value) == {"codec", "blob"} and isinstance(value.get("blob"), str):

‎src/durable_workflow/worker.py‎

Lines changed: 1 addition & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -98,10 +98,8 @@
9898
Command,
9999
FailWorkflow,
100100
LocalActivityExecutionAborted,
101-
NexusServiceCall,
102101
PreparedLocalActivityCall,
103102
RecordLocalActivity,
104-
RecordSideEffect,
105103
ReplayOutcome,
106104
ScheduleActivity,
107105
StartChildWorkflow,
@@ -166,7 +164,6 @@
166164
}
167165
_WORKFLOW_TASK_COMPLETION_MAX_ATTEMPTS = 3
168166
_WORKFLOW_TASK_COMPLETION_RETRY_DELAYS = (0.05, 0.2)
169-
_WORKFLOW_TASK_NEXUS_RESOLUTION_LIMIT = 100
170167
_LOCAL_ACTIVITY_REPORT_LIMIT = 1000
171168
_WORKER_WORKFLOW_FINGERPRINTS: dict[tuple[str, str], str] = {}
172169
_P = ParamSpec("_P")
@@ -1428,86 +1425,6 @@ async def call_interceptor(
14281425

14291426
return await handler(context)
14301427

1431-
async def _resolve_workflow_nexus_commands(
1432-
self,
1433-
workflow_cls: type,
1434-
history: list[dict[str, Any]],
1435-
start_input: list[Any],
1436-
task: dict[str, Any],
1437-
*,
1438-
payload_codec: str | None,
1439-
command_codec: str,
1440-
initial_commands: list[Command],
1441-
) -> list[Command]:
1442-
events = list(history)
1443-
commands: list[Command] = list(initial_commands)
1444-
recorded_commands: list[RecordSideEffect] = []
1445-
1446-
for _ in range(_WORKFLOW_TASK_NEXUS_RESOLUTION_LIMIT):
1447-
nexus_commands = [command for command in commands if isinstance(command, NexusServiceCall)]
1448-
if not nexus_commands:
1449-
return recorded_commands + commands
1450-
if len(nexus_commands) != len(commands):
1451-
raise RuntimeError("Nexus service calls must be yielded one at a time")
1452-
1453-
for command in nexus_commands:
1454-
result = await self.client.execute_nexus_operation(
1455-
command.endpoint_name,
1456-
command.service_name,
1457-
command.operation_name,
1458-
command.arguments,
1459-
mode=command.mode,
1460-
wait_for=command.wait_for,
1461-
wait_timeout_seconds=command.wait_timeout_seconds,
1462-
idempotency_key=command.idempotency_key,
1463-
payload_codec=command.payload_codec,
1464-
caller_namespace=command.caller_namespace,
1465-
target_namespace=command.target_namespace,
1466-
caller_workflow_instance_id=_string_or_none(task.get("workflow_id")),
1467-
caller_workflow_run_id=_string_or_none(task.get("run_id")),
1468-
service_sdk_language=command.service_sdk_language,
1469-
artifact_tuple=command.artifact_tuple,
1470-
published_artifact_worker_execution=command.published_artifact_worker_execution,
1471-
target_workflow_instance_id=command.target_workflow_instance_id,
1472-
target_workflow_run_id=command.target_workflow_run_id,
1473-
connection=command.connection,
1474-
queue=command.queue,
1475-
business_key=command.business_key,
1476-
labels=command.labels,
1477-
memo=command.memo,
1478-
search_attributes=command.search_attributes,
1479-
duplicate_start_policy=command.duplicate_start_policy,
1480-
raise_on_failure=False,
1481-
)
1482-
recorded_payload = result.to_recorded_payload()
1483-
recorded_commands.append(RecordSideEffect(recorded_payload))
1484-
events.append({
1485-
"event_type": "SideEffectRecorded",
1486-
"payload": {
1487-
"payload_codec": command_codec,
1488-
"result": serializer.encode(recorded_payload, codec=command_codec),
1489-
},
1490-
})
1491-
1492-
outcome = replay(
1493-
workflow_cls,
1494-
events,
1495-
start_input,
1496-
workflow_id=task.get("workflow_id"),
1497-
run_id=_string_or_none(task.get("run_id")) or "",
1498-
workflow_command_id=(
1499-
_string_or_none(task.get("workflow_command_id"))
1500-
or _string_or_none(task.get("task_id"))
1501-
),
1502-
payload_codec=payload_codec,
1503-
external_storage=self.external_storage,
1504-
external_storage_cache=self.external_storage_cache,
1505-
cancel_requested=bool(task.get("cancel_requested", False)),
1506-
)
1507-
commands = list(outcome.commands)
1508-
1509-
raise RuntimeError("workflow yielded too many consecutive Nexus service calls")
1510-
15111428
async def _renew_local_workflow_lease(self, task: dict[str, Any]) -> None:
15121429
if self._cooperative_cancellation_supported and self._local_activity_shutdown.is_set():
15131430
raise LocalActivityExecutionAborted("worker shutdown abandoned its local workflow claim")
@@ -2629,45 +2546,7 @@ def execute_local(command: RecordLocalActivity) -> Any:
26292546
log.warning("failed to report replay failure: %s", fe)
26302547
return None
26312548

2632-
try:
2633-
workflow_commands = await self._resolve_workflow_nexus_commands(
2634-
cls,
2635-
list(history),
2636-
start_input,
2637-
task,
2638-
payload_codec=codec,
2639-
command_codec=command_codec,
2640-
initial_commands=list(outcome.commands),
2641-
)
2642-
except Exception as e:
2643-
log.exception("workflow Nexus service call resolution failed")
2644-
last_local = max(
2645-
(index for index, command in enumerate(outcome.commands)
2646-
if isinstance(command, RecordLocalActivity) and command.outcome is not None),
2647-
default=-1,
2648-
)
2649-
if last_local >= 0:
2650-
workflow_commands = [
2651-
*outcome.commands[:last_local + 1],
2652-
FailWorkflow(
2653-
f"Nexus resolution after local activity failed: {e}",
2654-
exception_type=type(e).__name__,
2655-
non_retryable=True,
2656-
),
2657-
]
2658-
else:
2659-
try:
2660-
await self.client.fail_workflow_task(
2661-
task_id=task_id,
2662-
lease_owner=self.worker_id,
2663-
workflow_task_attempt=attempt,
2664-
message=f"workflow Nexus service call resolution failed: {e}",
2665-
failure_type=type(e).__name__,
2666-
stack_trace=traceback.format_exc(),
2667-
)
2668-
except Exception as fe:
2669-
log.warning("failed to report workflow Nexus resolution failure: %s", fe)
2670-
return None
2549+
workflow_commands = list(outcome.commands)
26712550

26722551
if (
26732552
any(isinstance(command, UpsertMemo) for command in workflow_commands)

0 commit comments

Comments
 (0)