3737from types import FunctionType
3838from typing import Annotated , Any , Concatenate , Literal , ParamSpec , TypeVar , Union , get_args , get_origin , get_type_hints
3939
40+ import httpx
41+
4042from . import serializer
4143from ._activity_lease import activity_lease_renewal_supported , current_activity_authority
4244from ._activity_process import CallbackFailure , CallbackInvocation , CallbackProcessLost , SupervisedCallback
@@ -1169,6 +1171,10 @@ def __init__(
11691171 self ._in_flight : set [asyncio .Task [Any ]] = set ()
11701172 self ._poller_tasks : set [asyncio .Task [Any ]] = set ()
11711173 self ._registered = False
1174+ self ._deregistration_fenced = False
1175+ self ._registration_token : str | None = None
1176+ self ._registration_worker_id : str | None = None
1177+ self ._registration_namespace : object = None
11721178 self ._run_started = False
11731179 self ._registration_done = asyncio .Event ()
11741180 self ._shutdown_task : asyncio .Task [None ] | None = None
@@ -1295,6 +1301,18 @@ async def _register(self) -> None:
12951301 _validate_server_compatibility (info )
12961302 protocol = info .get ("worker_protocol" )
12971303 server_capabilities = protocol .get ("server_capabilities" ) if isinstance (protocol , Mapping ) else None
1304+ deregistration = (
1305+ server_capabilities .get ("worker_deregistration_fencing" )
1306+ if isinstance (server_capabilities , Mapping ) else None
1307+ )
1308+ self ._deregistration_fenced = isinstance (deregistration , Mapping ) and deregistration .get ("supported" ) is True
1309+ if isinstance (deregistration , Mapping ) and self ._deregistration_fenced and (
1310+ deregistration .get ("schema" ) != "durable-workflow.v2.worker-deregistration.v1"
1311+ or deregistration .get ("endpoint" ) != "/worker/registrations/{workerId}/deregister"
1312+ or type (deregistration .get ("receipt_retention_seconds" )) is not int
1313+ or deregistration ["receipt_retention_seconds" ] <= 0
1314+ ):
1315+ raise RuntimeError ("Server compatibility error: invalid worker_deregistration_fencing contract" )
12981316 sticky_support = (
12991317 server_capabilities .get ("sticky_execution" ) if isinstance (server_capabilities , Mapping ) else None
13001318 )
@@ -1449,6 +1467,20 @@ async def _register(self) -> None:
14491467 task_slots = self ._current_task_slots (),
14501468 process_metrics = self ._current_process_metrics (),
14511469 )
1470+ if self ._deregistration_fenced :
1471+ token = ack .get ("registration_token" ) if isinstance (ack , dict ) else None
1472+ if (
1473+ not isinstance (token , str ) or len (token ) != 32
1474+ or any (character not in "0123456789abcdef" for character in token )
1475+ or ack .get ("worker_id" ) != self .worker_id or ack .get ("registered" ) is not True
1476+ ):
1477+ raise RuntimeError (
1478+ "Server returned an invalid registration_token acknowledgement; "
1479+ "registration may remain active and unfenced cleanup is unsafe"
1480+ )
1481+ self ._registration_token = token
1482+ self ._registration_worker_id = self .worker_id
1483+ self ._registration_namespace = getattr (self .client , "namespace" , None )
14521484 self ._registered = True
14531485 # Adapt to the server-advertised cadence when present so a cluster
14541486 # can pin the worker fleet's heartbeat beat without each worker
@@ -4567,8 +4599,9 @@ def _track_worker_session_from_task(self, task: dict[str, Any]) -> None:
45674599 async def stop (self ) -> None :
45684600 """Stop polling, drain tasks, and remove the worker registration.
45694601
4570- Repeated calls share one shutdown result. A failed deregistration is
4571- therefore propagated to every caller without sending another DELETE.
4602+ Repeated calls share one shutdown result. Generation-fenced recovery
4603+ stays within the original shutdown deadline. Unfenced deregistration
4604+ makes one attempt and does not retry an uncertain outcome.
45724605 """
45734606 self ._stop .set ()
45744607 if self ._shutdown_task is None :
@@ -4664,8 +4697,9 @@ async def _shutdown(self) -> None:
46644697 log .warning ("worker session %s close failed: %s" , session .options .session_id , error )
46654698
46664699 if self ._registered :
4667- await self .client . deregister_worker_registration ( self . worker_id )
4700+ await self ._deregister_before_deadline ( deadline )
46684701 self ._registered = False
4702+ self ._registration_token = None
46694703 log .info ("worker %s deregistered" , self .worker_id )
46704704 self ._sticky_cache .clear ()
46714705 payload_observers = getattr (self .client , "_workflow_payload_observers" , None )
@@ -4676,3 +4710,62 @@ async def _shutdown(self) -> None:
46764710 @staticmethod
46774711 def _remaining_shutdown_time (deadline : float ) -> float :
46784712 return max (0.0 , deadline - asyncio .get_running_loop ().time ())
4713+
4714+ def _check_deregistration_identity (self ) -> None :
4715+ if self ._deregistration_fenced and self ._registration_token is None :
4716+ raise RuntimeError ("worker registration token is missing; registration removal is unconfirmed" )
4717+ if self ._registration_worker_id is not None and (
4718+ self .worker_id != self ._registration_worker_id
4719+ or getattr (self .client , "namespace" , None ) != self ._registration_namespace
4720+ ):
4721+ raise RuntimeError ("worker registration identity changed; registration removal is unconfirmed" )
4722+
4723+ async def _deregistration_attempt (self , remaining : float ) -> tuple [float | None , Exception ] | None :
4724+ kwargs : dict [str , Any ] = {}
4725+ if self ._deregistration_fenced :
4726+ kwargs = {"registration_token" : self ._registration_token , "timeout" : remaining }
4727+ if isinstance (self ._registration_namespace , str ):
4728+ kwargs ["expected_namespace" ] = self ._registration_namespace
4729+ try :
4730+ await asyncio .wait_for (
4731+ self .client .deregister_worker_registration (self .worker_id , ** kwargs ),
4732+ timeout = remaining ,
4733+ )
4734+ except asyncio .TimeoutError as error :
4735+ raise RuntimeError (
4736+ "worker shutdown timed out during deregistration; registration removal is unconfirmed"
4737+ ) from error
4738+ except ServerError as error :
4739+ retry_delay = (
4740+ error .worker_deregistration_retry_delay (self .worker_id , self ._registration_token )
4741+ if self ._deregistration_fenced and self ._registration_token is not None else None
4742+ )
4743+ if retry_delay is None :
4744+ raise
4745+ return float (retry_delay ), error
4746+ except (httpx .NetworkError , httpx .TimeoutException , httpx .RemoteProtocolError ) as error :
4747+ if not self ._deregistration_fenced :
4748+ raise
4749+ return None , error
4750+ return None
4751+
4752+ async def _deregister_before_deadline (self , deadline : float ) -> None :
4753+ attempt = 0
4754+ while True :
4755+ self ._check_deregistration_identity ()
4756+ remaining = self ._remaining_shutdown_time (deadline )
4757+ if remaining <= 0 :
4758+ raise RuntimeError ("worker shutdown deadline exhausted; registration removal is unconfirmed" )
4759+ retry = await self ._deregistration_attempt (remaining )
4760+ if retry is None :
4761+ return
4762+ retry_delay , failure = retry
4763+ delay = min (1.0 , 0.1 * 2 ** min (attempt , 4 )) if retry_delay is None else retry_delay
4764+ if delay >= self ._remaining_shutdown_time (deadline ):
4765+ raise RuntimeError (
4766+ "worker shutdown cannot retry within its deadline; registration removal is unconfirmed"
4767+ ) from failure
4768+ if attempt == 0 :
4769+ log .warning ("worker deregistration is unconfirmed; retrying the fenced request within its deadline" )
4770+ attempt += 1
4771+ await asyncio .sleep (delay )
0 commit comments