Skip to content

Commit bba6661

Browse files
committed
improve swarm instruction chain
1 parent 812d96c commit bba6661

16 files changed

Lines changed: 1031 additions & 16 deletions

File tree

‎ajet/copilot/job.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,14 @@ class AgentJetJob:
6767
whose ``num_repeat`` episodes do *not* all share the same reward. Tasks with uniform
6868
reward (e.g. all 0 or all 1) produce zero advantage under GRPO and are skipped —
6969
useful when the dataset contains many too-easy or too-hard prompts.
70+
- "rollout_until_any_client_agree_sync_weight": defer the stop decision to the swarm
71+
clients themselves. Stops as soon as **any** active swarm client invokes
72+
``SwarmClient.agree_sync_weight()``. A client is "active" once it has successfully
73+
``end_episode``'d at least one rewarded (non-abort) episode since the last weight
74+
sync, and falls off the active list after 10 minutes of no chat-completion or
75+
``begin_episode`` activity.
76+
- "rollout_until_all_clients_agree_sync_weight": like the above, but stops only when
77+
**every** active swarm client has agreed (and there is at least one active client).
7078
max_env_worker: an estimation about how many episodes will be running in parallel (all swarm clients combined).
7179
backbone: Training backbone framework (e.g., 'verl').
7280
max_prompt_length: Maximum token length for input prompts (token length before the first llm-generated token, default 3000).

‎ajet/default_config/ajet_default.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,12 @@ ajet:
345345
# "rollout_until_finish_enough_non_dummy_tasks":
346346
# AgentJet will identify the **task_id** of each episode, and stop when it has collected [>= ajet.data.train_batch_size] unique & FINISHED & NON-DUMMY **task_id**.
347347
# (Hint: a **task_id** is considered "NON-DUMMY" at least one of **episodes** of **task_id** has **different** reward value.)
348+
# "rollout_until_any_client_agree_sync_weight":
349+
# AgentJet defers the stop decision to swarm clients: stop as soon as ANY active swarm client has called `SwarmClient.agree_sync_weight()`.
350+
# (Hint: a swarm client becomes "active" once it has successfully `end_episode`'d a rewarded (non-abort) episode since the last weight sync,
351+
# and falls off the active list if it does no chat-completion / begin_episode for 10 minutes.)
352+
# "rollout_until_all_clients_agree_sync_weight":
353+
# Like the above, but stop only when EVERY active swarm client has agreed (and there is at least one active client).
348354
swarm_mode_sample_collection_method: "rollout_until_finish_enough_tasks"
349355
swarm_mode_sample_collection_max_cached_episodes: 9999
350356

‎ajet/default_config/ajet_swarm_default.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ ajet:
5353
# "rollout_until_finish_enough_non_dummy_tasks":
5454
# AgentJet will identify the **task_id** of each episode, and stop when it has collected [>= ajet.data.train_batch_size] unique & FINISHED & NON-DUMMY **task_id**.
5555
# (Hint: a **task_id** is considered "NON-DUMMY" at least one of **episodes** of **task_id** has **different** reward value.)
56+
# "rollout_until_any_client_agree_sync_weight":
57+
# AgentJet defers the stop decision to swarm clients: stop as soon as ANY active swarm client has called `SwarmClient.agree_sync_weight()`.
58+
# (Hint: a swarm client becomes "active" once it has successfully `end_episode`'d a rewarded (non-abort) episode since the last weight sync,
59+
# and falls off the active list if it does no chat-completion / begin_episode for 10 minutes.)
60+
# "rollout_until_all_clients_agree_sync_weight":
61+
# Like the above, but stop only when EVERY active swarm client has agreed (and there is at least one active client).
5662
swarm_mode_sample_collection_method: "rollout_until_finish_enough_tasks"
5763

5864
data:

‎ajet/task_rollout/native_parallel_worker.py‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@
2626
from ajet.context_tracker.single_agent_tracking import SingleAgentContextTracker
2727
from ajet.tuner_lib.experimental.interchange_utils import (
2828
http_change_engine_status,
29-
http_update_rollout_pool_information,
29+
http_update_rollout_pool_information_and_fetch_instruction,
3030
CurrentBatchRolloutPoolInformation,
31+
SwarmClientInstruction,
3132
)
3233

3334

@@ -292,6 +293,14 @@ def rollout_swarm( # noqa: C901
292293
completed_task_id_map_ct: Dict[str, List[SingleAgentContextTracker]] = IterationSafeDict()
293294
executor_lock = threading.Lock()
294295

296+
accept_client_control = ("client" in self.config.ajet.swarm_mode_sample_collection_method)
297+
if accept_client_control:
298+
# Latest active-client / agreed-sync-weight snapshot from the swarm server. Refreshed on every pool-information update;
299+
# consumed by the `rollout_until_*_agree_sync_weight` stop conditions.
300+
latest_swarm_client_instructions: Dict[str, SwarmClientInstruction | None] = {"swarm_clients": None}
301+
else:
302+
latest_swarm_client_instructions = None
303+
295304
# count tasks to see whether we have reach the finish line for next weight update
296305
def count_tasks(completed_task_id_map_ct):
297306
total_completed_episodes = 0
@@ -340,6 +349,20 @@ def enough_finished_task_stop_condition(completed_task_id_map_ct) -> bool:
340349
completed_task_id_map_ct.clear()
341350
return (total_completed_tasks >= n_batch_task)
342351

352+
def any_client_agree_sync_weight_stop_condition(completed_task_id_map_ct) -> bool:
353+
# ajet.swarm_mode_sample_collection_method == "rollout_until_any_client_agree_sync_weight"
354+
instr = latest_swarm_client_instructions["swarm_clients"]
355+
if instr is None:
356+
return False
357+
return any(c.allowed_sync_weight for c in instr.active_clients)
358+
359+
def all_clients_agree_sync_weight_stop_condition(completed_task_id_map_ct) -> bool:
360+
# ajet.swarm_mode_sample_collection_method == "rollout_until_all_clients_agree_sync_weight"
361+
instr = latest_swarm_client_instructions["swarm_clients"]
362+
if instr is None or not instr.active_clients:
363+
return False
364+
return all(c.allowed_sync_weight for c in instr.active_clients)
365+
343366
def enough_non_dummy_task_stop_condition(completed_task_id_map_ct) -> bool:
344367
# ajet.swarm_mode_sample_collection_method == "rollout_until_finish_enough_non_dummy_tasks"
345368
counts = count_tasks(completed_task_id_map_ct)
@@ -372,6 +395,10 @@ def enough_non_dummy_task_stop_condition(completed_task_id_map_ct) -> bool:
372395
stop_condition = enough_finished_task_stop_condition
373396
elif self.config.ajet.swarm_mode_sample_collection_method == "rollout_until_finish_enough_non_dummy_tasks":
374397
stop_condition = enough_non_dummy_task_stop_condition
398+
elif self.config.ajet.swarm_mode_sample_collection_method == "rollout_until_any_client_agree_sync_weight":
399+
stop_condition = any_client_agree_sync_weight_stop_condition
400+
elif self.config.ajet.swarm_mode_sample_collection_method == "rollout_until_all_clients_agree_sync_weight":
401+
stop_condition = all_clients_agree_sync_weight_stop_condition
375402
else:
376403
logger.error(f"Invalid swarm_mode_sample_collection_method: {self.config.ajet.swarm_mode_sample_collection_method}, fallback to default method: rollout_until_finish_enough_tasks")
377404
stop_condition = enough_finished_task_stop_condition
@@ -442,9 +469,16 @@ def update_rollout_result_array_preview(observation_window, completed_task_id_ma
442469
buffer += f"Total completed tasks: {counts['total_completed_tasks']} (target {n_batch_task})\n"
443470
buffer += f"Total completed non-dummy tasks: {counts['total_completed_non_dummy_tasks']} (target {n_batch_task})\n"
444471
buffer += f"Current stop condition: {self.config.ajet.swarm_mode_sample_collection_method}\n"
472+
if accept_client_control:
473+
sc_inst = latest_swarm_client_instructions["swarm_clients"]
474+
if sc_inst is not None:
475+
n_active = len(sc_inst.active_clients)
476+
n_agreed = sum(1 for c in sc_inst.active_clients if c.allowed_sync_weight)
477+
buffer += f"Active clients: {n_active} (agreed: {n_agreed})\n"
445478
observation_window["info"][-1] = buffer
446479

447-
# Update rollout pool information via API
480+
# Update rollout pool information via API and pull the latest
481+
# active-client / agreed-sync-weight instruction from the server.
448482
pool_info = CurrentBatchRolloutPoolInformation(
449483
sample_collection_method=self.config.ajet.swarm_mode_sample_collection_method,
450484
completed_episodes=counts['total_completed_episodes'],
@@ -457,7 +491,10 @@ def update_rollout_result_array_preview(observation_window, completed_task_id_ma
457491
completed_tasks_details=completed_tasks_details,
458492
completed_tasks_rewards=completed_tasks_rewards,
459493
)
460-
http_update_rollout_pool_information(self.config, pool_info)
494+
if accept_client_control:
495+
instruction = http_update_rollout_pool_information_and_fetch_instruction(self.config, pool_info)
496+
if instruction is not None:
497+
latest_swarm_client_instructions["swarm_clients"] = instruction
461498
return
462499

463500
update_rollout_result_array_preview(observation_window, completed_task_id_map_ct)

‎ajet/tuner_lib/experimental/interchange_utils.py‎

Lines changed: 145 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,138 @@ class VerboseLogsResponse(BaseModel):
118118
entries: List[VerboseLogEntry] = []
119119

120120

121+
class AgreeSyncWeightRequest(BaseModel):
122+
client_uuid: str
123+
124+
125+
class ActiveSwarmClient(BaseModel):
126+
"""Server-tracked record for one active swarm client.
127+
128+
A swarm client enters this list once it has successfully `end_episode`'d
129+
a rewarded (non-abort) episode since the last weight sync, and falls off
130+
after `CLIENT_ACTIVE_TIMEOUT` seconds of no chat-completion /
131+
`begin_episode` activity. The whole list is reset whenever the engine
132+
leaves ROLLING/ROLLING_POST.
133+
134+
Used both as the swarm server's authoritative storage (single
135+
`shared_mem_dict["active_swarm_clients"]: List[ActiveSwarmClient]` key)
136+
and as the wire payload sent back to the trainer in
137+
`SwarmClientInstruction`. Add future per-client signals (e.g.
138+
`requested_pause`, custom metrics) here -- pydantic field defaults keep
139+
the wire format backwards-compatible across server/trainer versions.
140+
141+
Fields:
142+
client_uuid: the client_uuid as generated in `SwarmClient.__init__`.
143+
last_activity_at: unix timestamp of the most recent chat-completion,
144+
`begin_episode`, or `end_episode` from this client. Used by the
145+
server's expiry sweep.
146+
allowed_sync_weight: True iff this client has explicitly agreed to
147+
the next weight sync via `SwarmClient.agree_sync_weight()`.
148+
"""
149+
client_uuid: str
150+
last_activity_at: float
151+
allowed_sync_weight: bool = False
152+
153+
154+
class SwarmClientInstruction(BaseModel):
155+
"""Server -> trainer instruction returned alongside pool-info updates.
156+
157+
Fields:
158+
active_clients: list of `ActiveSwarmClient` records, one per
159+
currently active client.
160+
161+
Example wire payload:
162+
```json
163+
{
164+
"active_clients": [
165+
{"client_uuid": "9f3c-...-aaaa", "last_activity_at": 1746513900.1, "allowed_sync_weight": true},
166+
{"client_uuid": "9f3c-...-bbbb", "last_activity_at": 1746513912.4, "allowed_sync_weight": false},
167+
{"client_uuid": "9f3c-...-cccc", "last_activity_at": 1746513918.7, "allowed_sync_weight": false}
168+
]
169+
}
170+
```
171+
172+
Example trainer-side use (matches DynamicRolloutManager.rollout_swarm):
173+
```python
174+
# rollout_until_any_client_agree_sync_weight
175+
if any(c.allowed_sync_weight for c in instr.active_clients):
176+
stop()
177+
178+
# rollout_until_all_clients_agree_sync_weight
179+
if instr.active_clients and all(
180+
c.allowed_sync_weight for c in instr.active_clients
181+
):
182+
stop()
183+
```
184+
185+
For the payload above:
186+
- "any" stop-condition evaluates True (one client agreed).
187+
- "all" stop-condition evaluates False (two of three not yet agreed).
188+
"""
189+
active_clients: List[ActiveSwarmClient] = []
190+
191+
192+
# Active-client tracking timeout (seconds): a client falls off the active list
193+
# if it has done no chat-completion or begin_episode call within this window.
194+
CLIENT_ACTIVE_TIMEOUT = 10 * 60
195+
196+
197+
# --------------------------------------------------------------------
198+
# active-client tracking helpers
199+
# --------------------------------------------------------------------
200+
# All active-client state lives behind a single shared_mem_dict key:
201+
# "active_swarm_clients": List[ActiveSwarmClient]
202+
# (See `ActiveSwarmClient` for field semantics and lifecycle.) The helpers
203+
# below are imported by the swarm server's FastAPI routes and by the
204+
# OAI-mode chat-completion handler.
205+
206+
207+
def _refresh_client_activity(client_uuid: str, shared_mem_dict) -> None:
208+
"""If client is in the active list, refresh its last-activity timestamp.
209+
210+
Called on chat-completion and begin_episode (claim_episode). Does NOT
211+
add the client to the list -- only end_episode (success, non-abort) does.
212+
"""
213+
if not client_uuid:
214+
return
215+
clients: List[ActiveSwarmClient] = list(shared_mem_dict.get("active_swarm_clients", []))
216+
for i, c in enumerate(clients):
217+
if c.client_uuid == client_uuid:
218+
clients[i] = c.model_copy(update={"last_activity_at": time.time()})
219+
shared_mem_dict["active_swarm_clients"] = clients
220+
return
221+
222+
223+
def _register_active_client(client_uuid: str, shared_mem_dict) -> None:
224+
"""Add client to the active list (idempotent) and refresh its timestamp."""
225+
if not client_uuid:
226+
return
227+
clients: List[ActiveSwarmClient] = list(shared_mem_dict.get("active_swarm_clients", []))
228+
now = time.time()
229+
for i, c in enumerate(clients):
230+
if c.client_uuid == client_uuid:
231+
clients[i] = c.model_copy(update={"last_activity_at": now})
232+
shared_mem_dict["active_swarm_clients"] = clients
233+
return
234+
clients.append(ActiveSwarmClient(client_uuid=client_uuid, last_activity_at=now))
235+
shared_mem_dict["active_swarm_clients"] = clients
236+
237+
238+
def _expire_inactive_clients(shared_mem_dict) -> None:
239+
"""Drop clients whose last activity is older than CLIENT_ACTIVE_TIMEOUT."""
240+
now = time.time()
241+
clients: List[ActiveSwarmClient] = list(shared_mem_dict.get("active_swarm_clients", []))
242+
if not clients:
243+
return
244+
kept = [c for c in clients if (now - c.last_activity_at) <= CLIENT_ACTIVE_TIMEOUT]
245+
if len(kept) != len(clients):
246+
shared_mem_dict["active_swarm_clients"] = kept
247+
248+
249+
def _reset_active_client_tracking(shared_mem_dict) -> None:
250+
"""Clear all active-client state."""
251+
shared_mem_dict["active_swarm_clients"] = []
252+
121253

122254
DEBUG = False
123255
# DEBUG = True
@@ -233,24 +365,34 @@ def http_push_verbose_log(message: str, tag: str = "", config=None):
233365
logger.warning(f"Failed to push verbose log: {e}")
234366

235367

236-
def http_update_rollout_pool_information(config, pool_info: CurrentBatchRolloutPoolInformation):
368+
def http_update_rollout_pool_information_and_fetch_instruction(
369+
config, pool_info: CurrentBatchRolloutPoolInformation
370+
) -> SwarmClientInstruction | None:
237371
"""
238-
Update the rollout pool information on the interchange server.
372+
Update the rollout pool information on the interchange server, and fetch
373+
the swarm server's view of currently-active clients and their
374+
agree-to-sync-weight state.
239375
240376
Args:
241377
config: The configuration object
242378
pool_info: CurrentBatchRolloutPoolInformation object with rollout statistics
379+
380+
Returns:
381+
SwarmClientInstruction with `active_clients` (List[ActiveSwarmClient]),
382+
or None if the request failed.
243383
"""
244384
try:
245385
resp = httpx.post(
246-
f"{get_interchange_server_url(config)}/update_current_batch_rollout_pool_information",
386+
f"{get_interchange_server_url(config)}/update_current_batch_rollout_pool_information_and_fetch_instruction",
247387
json=pool_info.model_dump(),
248388
timeout=5
249389
)
250390
resp.raise_for_status()
391+
return SwarmClientInstruction.model_validate(resp.json())
251392
except Exception as e:
252393
if DEBUG:
253394
logger.warning(f"Failed to update rollout pool information: {e}")
395+
return None
254396

255397

256398
def get_zmq_socket(config, episode_uuid: str, tag: str = ""):

‎ajet/tuner_lib/experimental/oai_model_server.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,7 @@ async def chat_completions(request: Request, authorization: str = Header(None)):
301301
# enable_swarm_mode
302302
if enable_swarm_mode:
303303
from ajet.tuner_lib.experimental.swarm_server import ep_key
304+
from ajet.tuner_lib.experimental.interchange_utils import _refresh_client_activity
304305
assert shared_mem_dict is not None
305306
assert shared_mem_dict_lock is not None
306307

@@ -319,6 +320,9 @@ async def chat_completions(request: Request, authorization: str = Header(None)):
319320
shared_mem_dict[ep_key(episode_uuid)] = es
320321
if es.episode_type == "eval":
321322
preserve_sampling_params = True
323+
# chat-completion counts as activity for keeping the owning client
324+
# in the swarm-server active list (no-op if it's not active yet).
325+
_refresh_client_activity(es.client_uuid, shared_mem_dict)
322326

323327
# For streaming, we process as non-streaming but return in streaming format
324328
original_stream = new_req.stream

0 commit comments

Comments
 (0)