From d399b0f211cd2e429974c1397b5b684abe31bca3 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 2 Apr 2026 16:51:24 +0000 Subject: [PATCH 01/58] Integrate ArcticRL --- verl/experimental/agent_loop/agent_loop.py | 9 ++++++--- verl/single_controller/ray/base.py | 15 ++++++++------- verl/trainer/config/ppo_trainer.yaml | 3 +++ verl/trainer/main_ppo.py | 15 +++++++++++++-- verl/trainer/ppo/ray_trainer.py | 10 ++++++++-- verl/utils/fsdp_utils.py | 13 +++++++++++++ verl/workers/engine_workers.py | 2 +- verl/workers/rollout/replica.py | 5 +++++ 8 files changed, 57 insertions(+), 15 deletions(-) diff --git a/verl/experimental/agent_loop/agent_loop.py b/verl/experimental/agent_loop/agent_loop.py index 8879960f128..8e7118db65e 100644 --- a/verl/experimental/agent_loop/agent_loop.py +++ b/verl/experimental/agent_loop/agent_loop.py @@ -918,13 +918,14 @@ def __init__( worker_group: RayWorkerGroup = None, rollout_resource_pool: RayResourcePool = None, reward_loop_worker_handles: list[ray.actor.ActorHandle] = None, - ): + **kwargs, + ): self.config = config self.rollout_config, self.model_config = _get_rollout_and_model_config(config) self.worker_group = worker_group self.rollout_resource_pool = rollout_resource_pool self.reward_loop_worker_handles = reward_loop_worker_handles - + self.kwargs = kwargs assert worker_group is not None or self.rollout_config.nnodes > 0, "nnodes must be > 0 in standalone mode" # for recipe to change @@ -941,9 +942,10 @@ async def create( worker_group: RayWorkerGroup = None, rollout_resource_pool: RayResourcePool = None, reward_loop_worker_handles: list[ray.actor.ActorHandle] = None, + **kwargs, ): """Create agent loop manager.""" - instance = cls(config, worker_group, rollout_resource_pool, reward_loop_worker_handles) + instance = cls(config, worker_group, rollout_resource_pool, reward_loop_worker_handles, **kwargs) await instance._initialize_llm_servers() await instance._init_global_load_balancer() await instance._init_agent_loop_workers() @@ -968,6 +970,7 @@ async def _initialize_llm_servers(self): config=self.rollout_config, model_config=self.model_config, gpus_per_node=self.rollout_config.n_gpus_per_node, + **self.kwargs, ) for replica_rank in range(num_replicas) ] diff --git a/verl/single_controller/ray/base.py b/verl/single_controller/ray/base.py index 2f6ee47064f..a7872b189a1 100644 --- a/verl/single_controller/ray/base.py +++ b/verl/single_controller/ray/base.py @@ -187,8 +187,9 @@ class ResourcePoolManager: resource_pool_spec: dict[str, list[int]] mapping: dict[int, str] resource_pool_dict: dict[str, RayResourcePool] = field(default_factory=dict) + gpu_resource_pool_dict: dict[str, RayResourcePool] = field(default_factory=dict) - def create_resource_pool(self): + def create_resource_pool(self, use_gpu: bool = True): """Create Ray resource pools for distributed training. Initializes resource pools based on the resource pool specification, @@ -202,10 +203,11 @@ def create_resource_pool(self): # For Megatron backend, we recommend using max_colocate_count>1 # that can utilize different WorkerGroup for differnt models resource_pool = RayResourcePool( - process_on_nodes=process_on_nodes, use_gpu=True, max_colocate_count=3, name_prefix=resource_pool_name + process_on_nodes=process_on_nodes, use_gpu=use_gpu, max_colocate_count=3, name_prefix=resource_pool_name ) self.resource_pool_dict[resource_pool_name] = resource_pool - + if use_gpu: + self.gpu_resource_pool_dict[resource_pool_name] = resource_pool self._check_resource_available() def get_resource_pool(self, role) -> RayResourcePool: @@ -214,7 +216,8 @@ def get_resource_pool(self, role) -> RayResourcePool: def get_n_gpus(self) -> int: """Get the number of gpus in this cluster.""" - return sum([n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes]) + process_on_gpu_nodes = [process_on_nodes for pool_name, process_on_nodes in self.resource_pool_spec.items() if pool_name in self.gpu_resource_pool_dict] + return sum([n_gpus for process_on_nodes in process_on_gpu_nodes for n_gpus in process_on_nodes]) def _check_resource_available(self): """Check if the resource pool can be satisfied in this ray cluster.""" @@ -226,9 +229,7 @@ def _check_resource_available(self): # check total required gpus can be satisfied total_available_gpus = sum(node_available_gpus.values()) - total_required_gpus = sum( - [n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes] - ) + total_required_gpus = self.get_n_gpus() if total_available_gpus < total_required_gpus: raise ValueError( f"Total available GPUs {total_available_gpus} is less than total desired GPUs {total_required_gpus}" diff --git a/verl/trainer/config/ppo_trainer.yaml b/verl/trainer/config/ppo_trainer.yaml index fd9b59862ae..2a0779cff1c 100644 --- a/verl/trainer/config/ppo_trainer.yaml +++ b/verl/trainer/config/ppo_trainer.yaml @@ -203,6 +203,9 @@ trainer: # mode: "auto", "enable", or "disable" use_legacy_worker_impl: auto + # whether to use arctic rl + use_arctic_rl: False + # profiler configs global_profiler: diff --git a/verl/trainer/main_ppo.py b/verl/trainer/main_ppo.py index 2c84374d245..262a318be8b 100644 --- a/verl/trainer/main_ppo.py +++ b/verl/trainer/main_ppo.py @@ -134,6 +134,10 @@ def add_actor_rollout_worker(self, config): actor_rollout_cls = ActorRolloutRefWorker ray_worker_group_cls = RayWorkerGroup + if config.trainer.get("use_arctic_rl", False): + from verl.workers.arctic_workers import ActorRolloutRefWorker + actor_rollout_cls = ActorRolloutRefWorker + lora_rank = config.actor_rollout_ref.model.get("lora", {}).get("rank", 0) if lora_rank <= 0: lora_rank = config.actor_rollout_ref.model.get("lora_rank", 0) @@ -340,7 +344,9 @@ def run(self, config): train_sampler = create_rl_sampler(config.data, train_dataset) # Initialize the PPO trainer. - trainer = RayPPOTrainer( + from verl.trainer.ppo.arctic_trainer import ArcticPPOTrainer + ppo_trainer_cls = RayPPOTrainer if not config.trainer.use_arctic_rl else ArcticPPOTrainer + trainer = ppo_trainer_cls( config=config, tokenizer=tokenizer, processor=processor, @@ -356,7 +362,12 @@ def run(self, config): trainer.init_workers() # Start the training process. - trainer.fit() + try: + trainer.fit() + finally: + # Ensure remote services shutdown gracefully + if hasattr(trainer, "destroy"): + trainer.destroy() def create_rl_dataset(data_paths, data_config, tokenizer, processor, is_train=True, max_samples: int = -1): diff --git a/verl/trainer/ppo/ray_trainer.py b/verl/trainer/ppo/ray_trainer.py index e178ffc143d..478bfd07908 100644 --- a/verl/trainer/ppo/ray_trainer.py +++ b/verl/trainer/ppo/ray_trainer.py @@ -309,6 +309,9 @@ def __init__( self.checkpoint_manager = None + self.wg_kwargs = {} + self.use_gpu = True + def _create_dataloader(self, train_dataset, val_dataset, collate_fn, train_sampler: Optional[Sampler]): """ Creates the train and validation dataloaders. @@ -682,7 +685,7 @@ def init_workers(self): 1. Ray resource pools from configuration 2. Worker groups for each role (actor, critic, etc.) """ - self.resource_pool_manager.create_resource_pool() + self.resource_pool_manager.create_resource_pool(use_gpu=self.use_gpu) self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} @@ -694,6 +697,7 @@ def init_workers(self): cls=self.role_worker_mapping[actor_role], config=self.config.actor_rollout_ref, role=str(actor_role), + **self.wg_kwargs, ) self.resource_pool_to_cls[actor_rollout_resource_pool][str(actor_role)] = actor_rollout_cls else: @@ -840,6 +844,7 @@ def init_workers(self): worker_group=self.actor_rollout_wg, rollout_resource_pool=actor_rollout_resource_pool, reward_loop_worker_handles=reward_loop_worker_handles, + **self.wg_kwargs, ) checkpoint_engine_config = omega_conf_to_dataclass(self.config.actor_rollout_ref.rollout.checkpoint_engine) self.checkpoint_manager = CheckpointEngineManager( @@ -1589,7 +1594,8 @@ def fit(self): metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) # TODO: implement actual tflpo and theoretical tflpo n_gpus = self.resource_pool_manager.get_n_gpus() - metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) + # To support serverless/tinker-like training, we need to support 0 GPUs training + metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=max(n_gpus, 1))) # compute variance proxy metrics gradient_norm = metrics.get("actor/grad_norm", None) metrics.update(compute_variance_proxy_metrics(batch=batch, gradient_norm=gradient_norm)) diff --git a/verl/utils/fsdp_utils.py b/verl/utils/fsdp_utils.py index 8bca54fa88c..11fc247d441 100644 --- a/verl/utils/fsdp_utils.py +++ b/verl/utils/fsdp_utils.py @@ -227,6 +227,19 @@ def load_fsdp_optimizer(optimizer, device_id): state[key] = value.to(device_id, non_blocking=True) +@torch.no_grad() +def get_fsdp_optimizer_devices(optimizer) -> list[torch.device]: + devices = set() + for param_group in optimizer.param_groups: + for param in param_group["params"]: + state = optimizer.state[param] + devices.add(param.device) + for key, value in state.items(): + if isinstance(value, torch.Tensor): + devices.add(value.device) + + return list(devices) + @contextmanager def meta_device_init(): """ diff --git a/verl/workers/engine_workers.py b/verl/workers/engine_workers.py index d0c065e4dfd..5367479a5bf 100644 --- a/verl/workers/engine_workers.py +++ b/verl/workers/engine_workers.py @@ -667,7 +667,7 @@ async def update_weights(self, global_steps: int = None): log_gpu_memory_usage("After update_weights", logger=logger) # 3. offload model to cpu - self.actor.engine.to("cpu", model=True, optimizer=False, grad=False) + self.actor.engine.to("cpu", model=self.actor.engine.is_param_offload_enabled, optimizer=False, grad=False) aggressive_empty_cache(force_sync=True) # 4. resume kv_cache diff --git a/verl/workers/rollout/replica.py b/verl/workers/rollout/replica.py index 969c6208083..2557eb74d7a 100644 --- a/verl/workers/rollout/replica.py +++ b/verl/workers/rollout/replica.py @@ -348,11 +348,16 @@ def _load_trtllm(): return TRTLLMReplica +def _load_arctic(): + from verl.workers.rollout.arctic_rollout.arctic_rollout import ArcticReplica + + return ArcticReplica # Register built-in types RolloutReplicaRegistry.register("vllm", _load_vllm) RolloutReplicaRegistry.register("sglang", _load_sglang) RolloutReplicaRegistry.register("trtllm", _load_trtllm) +RolloutReplicaRegistry.register("arctic", _load_arctic) # Original function for backward compatibility From 844bf7297ca34d8db3ccbda816efb90158ade671 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 2 Apr 2026 16:57:45 +0000 Subject: [PATCH 02/58] ArcticRL integgration --- examples/arctic_rl/run_arctic_gsm8k_grpo.sh | 86 ++ examples/arctic_rl/run_gsm8k_grpo.sh | 81 ++ verl/trainer/ppo/arctic_rl_client.py | 130 +++ verl/trainer/ppo/arctic_trainer.py | 153 +++ verl/workers/arctic_workers.py | 918 ++++++++++++++++++ .../rollout/arctic_rollout/__init__.py | 3 + .../rollout/arctic_rollout/arctic_rollout.py | 322 ++++++ 7 files changed, 1693 insertions(+) create mode 100755 examples/arctic_rl/run_arctic_gsm8k_grpo.sh create mode 100755 examples/arctic_rl/run_gsm8k_grpo.sh create mode 100644 verl/trainer/ppo/arctic_rl_client.py create mode 100644 verl/trainer/ppo/arctic_trainer.py create mode 100644 verl/workers/arctic_workers.py create mode 100644 verl/workers/rollout/arctic_rollout/__init__.py create mode 100644 verl/workers/rollout/arctic_rollout/arctic_rollout.py diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh new file mode 100755 index 00000000000..44cc760c708 --- /dev/null +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +# we want to make sure this runs on non-gpu client +export CUDA_VISIBLE_DEVICES= + +# BSZ=1024 +BSZ=2 +MBS=2 +UBS=2 +ROLL_N=2 +MAX_STEPS=1 +# LR=0 +LR=1e-6 +LOGGER=console +# LOGGER="['console','wandb']" +USE_KL_LOSS=True +# USE_KL_LOSS=False +# REMOVE_PADDING=True +REMOVE_PADDING=False +MODEL="Qwen/Qwen3-0.6B" +# STRATEGY="fsdp" +STRATEGY="fsdp2" +PYTHONUNBUFFERED=1 +HYDRA_FULL_ERROR=1 +USE_LEGACY_WORKER_IMPL=disable +NGPU_PER_NODE=1 +ROLLOUT_NAME=arctic # entry point into ArcticRL +USE_ARCTIC_RL=True + +experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_at${USE_ARCTIC_RL}" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=/code/shared/gsm8k/train.parquet \ + data.val_files=/code/shared/gsm8k/test.parquet \ + data.train_batch_size=${BSZ} \ + data.max_prompt_length=64 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + reward.num_workers=1 \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.model.path=${MODEL} \ + actor_rollout_ref.actor.optim.lr=${LR} \ + actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.strategy=${STRATEGY} \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.n=${ROLL_N} \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.ref.fsdp_config.param_offload=False \ + actor_rollout_ref.ref.strategy=${STRATEGY} \ + algorithm.use_kl_in_reward=False \ + trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ + trainer.use_arctic_rl=${USE_ARCTIC_RL} \ + trainer.critic_warmup=0 \ + trainer.logger=${LOGGER} \ + trainer.experiment_name=${experiment_name} \ + trainer.project_name='verl_arctic_grpo_gsm8k' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_training_steps=${MAX_STEPS} \ + trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log + + # trainer.total_training_steps=${MAX_STEPS} \ diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh new file mode 100755 index 00000000000..f65b0f55b24 --- /dev/null +++ b/examples/arctic_rl/run_gsm8k_grpo.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +# BSZ=1024 +BSZ=2 +MBS=2 +UBS=2 +ROLL_N=2 +MAX_STEPS=1 +# LR=0 +LR=1e-6 +LOGGER=console +# LOGGER="['console','wandb']" +USE_KL_LOSS=True +# USE_KL_LOSS=False +# REMOVE_PADDING=True +REMOVE_PADDING=False +MODEL="Qwen/Qwen3-0.6B" +# STRATEGY="fsdp" +STRATEGY="fsdp2" +PYTHONUNBUFFERED=1 +HYDRA_FULL_ERROR=1 +USE_LEGACY_WORKER_IMPL=disable +NGPU_PER_NODE=1 +ROLLOUT_NAME=vllm + +experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=/code/shared/gsm8k/train.parquet \ + data.val_files=/code/shared/gsm8k/test.parquet \ + data.train_batch_size=${BSZ} \ + data.max_prompt_length=64 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + reward.num_workers=1 \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.model.path=${MODEL} \ + actor_rollout_ref.actor.optim.lr=${LR} \ + actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.strategy=${STRATEGY} \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.n=${ROLL_N} \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.ref.fsdp_config.param_offload=False \ + actor_rollout_ref.ref.strategy=${STRATEGY} \ + algorithm.use_kl_in_reward=False \ + trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ + trainer.critic_warmup=0 \ + trainer.logger=${LOGGER} \ + trainer.experiment_name=${experiment_name} \ + trainer.project_name='verl_arctic_grpo_gsm8k' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_training_steps=${MAX_STEPS} \ + trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log + + # trainer.total_training_steps=${MAX_STEPS} \ diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py new file mode 100644 index 00000000000..8f7e74b21c9 --- /dev/null +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -0,0 +1,130 @@ +import torch +from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer +from deepspeed.utils import OnDevice +from dss_client.client import DSSInferenceClient, DSSTrainingClient, DSSLogProbClient +import ray +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy +from ray.util.placement_group import placement_group +from verl.workers.rollout.replica import TokenOutput +from tensordict import TensorDict +from typing import Any +from verl.utils.ray_utils import auto_await + +def create_arctic_rl_client(): + sched_pg = placement_group([{"GPU": 0, "CPU": 1}]) + arctic_rl_client = ray.remote( + num_cpus=0, + num_gpus=0, + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=sched_pg, + placement_group_capture_child_tasks=True, + ), + )(ArcticRLClient4VeRL).remote( + ) + + return arctic_rl_client + +def create_meta_model(name_or_path: str): + model_config = AutoConfig.from_pretrained(name_or_path) + with OnDevice(dtype=torch.float16, device='meta'): + meta_model = AutoModelForCausalLM.from_config(model_config) + return meta_model + +class ArcticRLClient4VeRL: + def __init__(self): + self.arctic_inference_client = DSSInferenceClient(dss_server_url="http://localhost:7000") + self.arctic_training_client = DSSTrainingClient(dss_server_url="http://localhost:7000") + self.arctic_log_prob_client = DSSLogProbClient(dss_server_url="http://localhost:7000") + + def initialize(self, model_name: str): + vllm_config = { + "temperature": 0.0, + "top_p": 1.0, + "top_k": 0, + "max_tokens": 1024, + "stop_sequences": [], + "stop_token_ids": [], + } + self.inference_engine = self.arctic_inference_client.initialize( + model_name=model_name, + vllm_config=vllm_config, + ) + self.log_prob_engine = self.arctic_log_prob_client.initialize( + model_name=model_name, + vllm_config=vllm_config, + ) + + ds_config = { + "train_micro_batch_size_per_gpu": 1, + "train_batch_size": 1, + "gradient_accumulation_steps": 1, + "sequence_parallel_size": 1, + "zero_optimization": { + "stage": 1, + }, + } + training_config = { + "optimizer": { + "lr": 0.0002, + "weight_decay": 0.0, + "betas": [0.9, 0.999], + }, + "lr_scheduler": {"warmup_ratio": 0.05}, + "training_horizon": 10, + "max_length": 8096, + "model_config": None, + "attn_implementation": "eager", + } + + self.training_engine = self.arctic_training_client.initialize( + model=create_meta_model(model_name), + ds_config=ds_config, + training_config=training_config) + + self.tokenizer = AutoTokenizer.from_pretrained(model_name) + + def generate(self, prompt_ids, sampling_params) -> TokenOutput: + prompts = [self.tokenizer.decode(prompt_ids)] + return self.inference_engine.generate( + prompts=prompts, + sampling_params=sampling_params, + ) + + def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict): + dss_batch_dict.update(post_process_inputs=post_process_inputs) + + # XXX: somehow we need to differentiate which model is this called on ref vs actor - at the moment it's always actor hardcoded + entropy, log_probs = self.training_engine.fwd_no_grad(**dss_batch_dict) + + # XXX: for some reason no_padding_2_padding expects a 1D tensor - not sure how it'll work for + # bs>1 + # I think it may have to do with tensor.is_nested - different path/logic + # so most likely we need to convert these 2 into TensorDict + if entropy is not None: + # prior_entropy_shape = entropy.shape + entropy = torch.tensor(entropy).squeeze() + if log_probs is not None: + # prior_log_probs_shape = log_probs.shape + log_probs = torch.tensor(log_probs).squeeze() + print(f"arctic_rl_client.compute_log_prob: {entropy.shape=}, {log_probs.shape=}") + return entropy, log_probs + + + def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): + + dss_batch_dict.update(post_process_inputs=post_process_inputs) + + #_ = self.training_engine.forward(**dss_batch_dict, post_process_inputs=post_process_inputs) + _ = self.training_engine.forward(**dss_batch_dict) + loss, metrics = self.training_engine.backward() + self.training_engine.step() + + print(f"arctic_rl_client.update_actor: {loss=}") + print(f"arctic_rl_client.update_actor: {metrics=}") + return loss.cpu().item(), metrics + + def destroy(self): + self.training_engine.destroy() + self.inference_engine.destroy() + return + diff --git a/verl/trainer/ppo/arctic_trainer.py b/verl/trainer/ppo/arctic_trainer.py new file mode 100644 index 00000000000..2d2a4e95f7b --- /dev/null +++ b/verl/trainer/ppo/arctic_trainer.py @@ -0,0 +1,153 @@ +import torch +from typing import Optional +from torch.utils.data import Dataset, Sampler +from verl.trainer.ppo.ray_trainer import RayPPOTrainer +from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup, ResourcePoolManager +from verl.workers.arctic_workers import ActorRolloutRefWorker +from verl.trainer.ppo.utils import Role, WorkerType +from omegaconf import OmegaConf +from verl.single_controller.ray.base import create_colocated_worker_cls +from verl.trainer.ppo.arctic_rl_client import create_arctic_rl_client + +def my_pdb(): + return + import pdb; pdb.set_trace() + +class ArcticPPOTrainer(RayPPOTrainer): + def __init__( + self, + config, + tokenizer, + role_worker_mapping: dict[Role, WorkerType], + resource_pool_manager: ResourcePoolManager, + ray_worker_group_cls: type[RayWorkerGroup] = RayWorkerGroup, + processor=None, + train_dataset: Optional[Dataset] = None, + val_dataset: Optional[Dataset] = None, + collate_fn=None, + train_sampler: Optional[Sampler] = None, + device_name=None, + ): + super().__init__(config=config, + tokenizer=tokenizer, + processor=processor, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + train_dataset=train_dataset, + val_dataset=val_dataset, + collate_fn=collate_fn, + train_sampler=train_sampler, + device_name=device_name) + + self.use_gpu = False + self.rl_client = create_arctic_rl_client() + self.rl_client.initialize.remote(model_name="Qwen/Qwen3-0.6B") + self.wg_kwargs["arctic_rl_client"] = self.rl_client + + + def init_workers(self): + super().init_workers() + return + # print(f"ArcticPPOTrainer.init_workers: {self.actor_rollout_wg=}") + # print(f"ArcticPPOTrainer.init_workers: {self.ref_policy_wg=}") + # print(f"ArcticPPOTrainer.init_workers: {self.async_rollout_manager=}") + # print(f"ArcticPPOTrainer.init_workers: {self.reward_loop_manager=}") + # print(f"ArcticPPOTrainer.init_workers: {self.checkpoint_manager=}") + + # self.resource_pool_manager.create_resource_pool(use_gpu=self.use_gpu) + + # self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} + + # # create actor and rollout + # actor_role = Role.ActorRolloutRef if Role.ActorRolloutRef in self.role_worker_mapping else Role.ActorRollout + # actor_rollout_resource_pool = self.resource_pool_manager.get_resource_pool(actor_role) + # actor_rollout_cls = RayClassWithInitArgs( + # cls=self.role_worker_mapping[actor_role], + # config=self.config.actor_rollout_ref, + # role=str(actor_role), + # ) + # self.resource_pool_to_cls[actor_rollout_resource_pool][str(actor_role)] = actor_rollout_cls + + # # create reference policy if needed + # # if self.use_reference_policy and Role.RefPolicy in self.role_worker_mapping: + # # resource_pool = self.resource_pool_manager.get_resource_pool(Role.RefPolicy) + # # ref_policy_cls = RayClassWithInitArgs( + # # self.role_worker_mapping[Role.RefPolicy], + # # config=self.config.actor_rollout_ref, + # # role=str(Role.RefPolicy), + # # ) + # # self.resource_pool_to_cls[resource_pool][str(Role.RefPolicy)] = ref_policy_cls + + # # initialize WorkerGroup + # # NOTE: if you want to use a different resource pool for each role, which can support different parallel size, + # # you should not use `create_colocated_worker_cls`. + # # Instead, directly pass different resource pool to different worker groups. + # # See https://github.com/volcengine/verl/blob/master/examples/ray/tutorial.ipynb for more information. + # all_wg = {} + # wg_kwargs = {} # Setting up kwargs for RayWorkerGroup + # if OmegaConf.select(self.config.trainer, "ray_wait_register_center_timeout") is not None: + # wg_kwargs["ray_wait_register_center_timeout"] = self.config.trainer.ray_wait_register_center_timeout + # if OmegaConf.select(self.config.global_profiler, "steps") is not None: + # wg_kwargs["profile_steps"] = OmegaConf.select(self.config.global_profiler, "steps") + # # Only require nsight worker options when tool is nsys + # if OmegaConf.select(self.config.global_profiler, "tool") == "nsys": + # assert ( + # OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") + # is not None + # ), "worker_nsight_options must be set when using nsys with profile_steps" + # wg_kwargs["worker_nsight_options"] = OmegaConf.to_container( + # OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") + # ) + # wg_kwargs["device_name"] = self.device_name + + # for resource_pool, class_dict in self.resource_pool_to_cls.items(): + # if not class_dict: + # continue + # worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) + # wg_dict = self.ray_worker_group_cls( + # resource_pool=resource_pool, + # ray_cls_with_init=worker_dict_cls, + # use_gpu=self.use_gpu, + # **wg_kwargs, + # ) + # spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) + # all_wg.update(spawn_wg) + + + # self.actor_rollout_wg = all_wg[str(actor_role)] + # self.actor_rollout_wg.init_model() + + # # create reward loop manager + # from verl.experimental.reward_loop import RewardLoopManager + + # # initalize reward loop manager + # # reward model (colocate or standalone): get resource_pool + # # no reward model: resource_pool = None + # resource_pool = self.resource_pool_manager.get_resource_pool(Role.RewardModel) if self.use_rm else None + # self.reward_loop_manager = RewardLoopManager( + # config=self.config, + # rm_resource_pool=resource_pool, + # ) + + # self.async_rollout_mode = True + # from verl.experimental.agent_loop import AgentLoopManager + + # # enable_agent_reward_loop = not self.use_rm or self.config.reward.reward_model.enable_resource_pool + # # reward_loop_worker_handles = self.reward_loop_manager.reward_loop_workers if enable_agent_reward_loop else None + # # self.async_rollout_manager = AgentLoopManager.create( + # # config=self.config, + # # worker_group=self.actor_rollout_wg, + # # rollout_resource_pool=actor_rollout_resource_pool, + # # reward_loop_worker_handles=reward_loop_worker_handles, + # # ) + + # self.ref_policy_wg = self.actor_rollout_wg + # self.checkpoint_manager = self.actor_rollout_wg + # self.async_rollout_manager = self.actor_rollout_wg + + + + def destroy(self): + # self.actor_rollout_wg.destroy() + self.rl_client.destroy.remote() \ No newline at end of file diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py new file mode 100644 index 00000000000..0882870c77e --- /dev/null +++ b/verl/workers/arctic_workers.py @@ -0,0 +1,918 @@ +from pathlib import Path +import torch +from verl.utils.ray_utils import auto_await +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.protocol import DataProto +from verl.single_controller.base import Worker +from verl.utils.profiler import DistProfiler, DistProfilerExtension +from verl.workers.engine_workers import ActorRolloutRefWorker as EngineActorRolloutRefWorker +from omegaconf import DictConfig +from tensordict import TensorDict +from dss_client.client import DSSInferenceClient, DSSTrainingClient +from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer +from deepspeed.utils import OnDevice +from verl.utils import tensordict_utils as tu +import os +import ray +from verl.utils.config import omega_conf_to_dataclass +from verl.utils.device import ( + get_device_id, + get_device_name, + get_nccl_backend, + get_torch_device, + set_expandable_segments, +) +from codetiming import Timer +import functools +import logging +import os +from contextlib import nullcontext +from functools import partial +from itertools import chain + +import torch +from codetiming import Timer +from omegaconf import DictConfig, open_dict +from tensordict import NonTensorData, TensorDict +from torch.distributed.device_mesh import init_device_mesh + +try: + from verl.workers.engine.mindspeed.transformer_impl import repatch +except ImportError: + repatch = None +from verl.checkpoint_engine import CheckpointEngineRegistry +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.utils import tensordict_utils as tu +from verl.utils.config import omega_conf_to_dataclass +from verl.utils.device import get_device_name, set_expandable_segments +from verl.utils.distributed import initialize_global_process_group_ray +from verl.utils.flops_counter import FlopsCounter +from verl.utils.memory_utils import aggressive_empty_cache +from verl.utils.metric.utils import Metric +from verl.utils.profiler import DistProfiler, DistProfilerExtension, ProfilerConfig, log_gpu_memory_usage +from verl.utils.py_functional import append_to_dict +from verl.utils.tensordict_utils import maybe_fix_3d_position_ids +from verl.utils.torch_functional import allgather_dict_into_dict +from verl.workers.config import ActorConfig, HFModelConfig, RolloutConfig, TrainingWorkerConfig +from verl.workers.rollout.base import BaseRollout, get_rollout_class +from verl.workers.utils.losses import ppo_loss +from torch import Tensor +from verl.workers.engine.utils import postprocess_batch_func + + + +def create_meta_model(name_or_path: str): + model_config = AutoConfig.from_pretrained(name_or_path) + with OnDevice(dtype=torch.float16, device='meta'): + meta_model = AutoModelForCausalLM.from_config(model_config) + return meta_model + + +DATA_PROTO_KEYS = ["gen_batch_output", "old_log_prob", "ref_log_prob", "compute_advantage", "actor_output"] +TENSOR_DICT_KEYS = ["full_log_prob", "full_ref_log_prob", "full_actor_output"] + +def load_dump_data(train_batch_size, roll_n) -> dict[str, DataProto]: + global_step = 1 + dump_data = {} + dump_path = os.path.join('/code/users/truwase/data/at_verl_dump', f'tbs{train_batch_size}_n{roll_n}') + dump_dir = Path(dump_path) + os.path.exists(dump_dir) + for key in DATA_PROTO_KEYS: + dump_data[key] = DataProto.load_from_disk(Path(dump_dir, f"{global_step}_{key}.pt")) + for key in TENSOR_DICT_KEYS: + dump_data[key] = torch.load(Path(dump_dir, f"{global_step}_{key}.pt"), weights_only=False) + + return dump_data + + +def prepare_model_inputs_remove_padding(micro_batch: TensorDict): + from verl.utils import tensordict_utils as tu + from verl.utils.dataset.dataset_utils import DatasetPadMode + from verl.utils.debug import log_gpu_memory_usage + from verl.utils.device import get_device_id, get_device_name + from verl.utils.model import extract_multi_modal_inputs + from verl.utils.torch_functional import logprobs_from_logits + import verl.utils.torch_functional as verl_F + + use_remove_padding = tu.get_non_tensor_data(data=micro_batch, key="use_remove_padding", default=True) + pad_mode = tu.get_non_tensor_data(data=micro_batch, key="pad_mode", default=DatasetPadMode.NO_PADDING) + use_fused_kernels = tu.get_non_tensor_data(data=micro_batch, key="use_fused_kernels", default=False) + temperature = micro_batch["temperature"] + temperature_item = temperature + if use_fused_kernels: + assert not isinstance(temperature, torch.Tensor), ( + "use_fused_kernels does not support per sample temperature yet" + ) + assert pad_mode == DatasetPadMode.NO_PADDING, f"pad_mode {pad_mode} not supported" + + multi_modal_inputs = extract_multi_modal_inputs(micro_batch.get("multi_modal_inputs", [])) + input_ids = micro_batch["input_ids"] + position_ids = micro_batch["position_ids"] + + if not isinstance(temperature, torch.Tensor): + temperature = torch.tensor([temperature] * input_ids.shape[0], device=input_ids.device) + + temperature = temperature.to(torch.float32) + assert temperature.shape[0] == input_ids.shape[0] + + # args used to get outputs + output_args = {} + + # support per sample temperature + # temperature (bsz,) + # input_ids (bsz, j1) + temperature_rmpad = verl_F.expand_as_nested(temperature, input_ids).values() # (total_nnz,) + temperature_rmpad = temperature_rmpad.unsqueeze(0) # (1, total_nnz) + + if pad_mode == DatasetPadMode.NO_PADDING: + input_ids_rmpad = input_ids.values().unsqueeze(0) # (1, total_nnz) + if position_ids.dim() == 3: + position_ids_rmpad = position_ids.values().unsqueeze(1) # (4, 1, total_nnz) + else: + position_ids_rmpad = position_ids.values().unsqueeze(0) # (1, total_nnz) + else: + raise NotImplementedError(f"pad_mode {pad_mode} not implemented") + + # for compute the log_prob + input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=1) # (1, total_nnz) + + # pad and slice the inputs if sp > 1 + + input_ids_rmpad_rolled = input_ids_rmpad_rolled.squeeze(0) # ((total_nnz / sp) + pad) + temperature_rmpad = temperature_rmpad.squeeze(0) + output_args["input_ids_rmpad_rolled"] = input_ids_rmpad_rolled + output_args["temperature_rmpad"] = temperature_rmpad + + # only pass input_ids and position_ids to enable flash_attn_varlen + + model_inputs = { + "input_ids": input_ids_rmpad, + "attention_mask": None, + "position_ids": position_ids_rmpad, + "labels": input_ids_rmpad, + } + + extra_args = {} + if use_fused_kernels: + extra_args["temperature"] = temperature_item + extra_args["return_dict"] = True + + model_inputs.update(multi_modal_inputs) + model_inputs.update(extra_args) + + return model_inputs, output_args + + + +def prepare_extra_inputs(data: TensorDict) -> dict: + pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) + batch_size = data["input_ids"].shape[0] + seq_len_effective = data["input_ids"].offsets().diff() + max_seq_len = max(seq_len_effective) + ready_input_ids = torch.nested.to_padded_tensor( + data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) + ) + ready_position_ids = torch.nested.to_padded_tensor( + data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) + ) + + extra_inputs = dict( + prompts=data["prompts"], + responses=data["responses"], + attention_mask=data["attention_mask"], + max_response_len=data["max_response_len"], + global_batch_size=data["global_batch_size"], + response_mask=data["response_mask"], + old_log_probs=data["old_log_probs"], + advantages=data["advantages"], + ref_log_prob=data["ref_log_prob"], + rollout_is_weights=data.get("rollout_is_weights", None), + batch_num_tokens=data["loss_mask"].sum(), + ready_input_ids=ready_input_ids, + ready_position_ids=ready_position_ids, + ready_labels=ready_input_ids, + cu_seqlens=data["input_ids"].offsets(), + ) + + + return extra_inputs + +def prepare_log_prob_extra_inputs(data: TensorDict) -> dict: + pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) + batch_size = data["input_ids"].shape[0] + seq_len_effective = data["input_ids"].offsets().diff() + max_seq_len = max(seq_len_effective) + + ready_input_ids = torch.nested.to_padded_tensor( + data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) + ) + ready_position_ids = torch.nested.to_padded_tensor( + data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) + ) + + extra_inputs = dict( + ready_input_ids=ready_input_ids, + ready_position_ids=ready_position_ids, + ready_labels=ready_input_ids, + cu_seqlens=data["input_ids"].offsets() + ) + + return extra_inputs + + +def rm_padding(data: TensorDict, tensor: Tensor) -> Tensor: + cu_seqlens = data["input_ids"].offsets() + seq_lengths = cu_seqlens.diff() # (bsz,) + starts = torch.zeros_like(seq_lengths, dtype=torch.int64) # (bsz,) + tensor = torch.nested.narrow(tensor, 1, starts, seq_lengths, layout=torch.jagged) + tensor = torch.cat([t for t in tensor.unbind()]) + tensor = torch.nested.nested_tensor_from_jagged(tensor, cu_seqlens) + return tensor + +def postprocess_log_prob_output(data: TensorDict, entropy: Tensor, log_probs: Tensor) -> TensorDict: + x_entropy = rm_padding(data, entropy) + x_log_probs = rm_padding(data, log_probs) + + print(f"postprocess_log_prob_output: {x_entropy.shape=} {x_log_probs.shape=} {entropy.shape=} {log_probs.shape=}") + + micro_entropy = [t.unsqueeze(0) for t in x_entropy.unbind()] + micro_log_probs = [t.unsqueeze(0) for t in x_log_probs.unbind()] + output_lst = [] + for i in range(len(micro_entropy)): + model_output = { + "entropy": micro_entropy[i], + "log_probs": micro_log_probs[i], + } + output_lst.append({ + "model_output": model_output, + "metrics": {}, + "loss": 0.0, + }) + + return postprocess_batch_func(output_lst=output_lst, indices=None, data=data) + + +class TrainingWorker(Worker, DistProfilerExtension): + """ + TrainingWorker provides a Tinker-like API (https://thinkingmachines.ai/tinker/) as a RayWorkerGroup + to a single controller. Currently, we only provide more coarse grained APIs, + and do not provide exact APIs as Tinker does. But this can be added in the future. + """ + + def __init__(self, config: TrainingWorkerConfig, actor_config: ActorConfig, arctic_rl_client): + Worker.__init__(self) + + from verl.workers.engine import BaseEngine, EngineRegistry + + #initialize_global_process_group_ray(timeout_second=None) + + self.config = config + self.actor_config = actor_config + + self.arctic_rl_client = arctic_rl_client + + self.model_config = self.config.model_config + self.engine_config = self.config.engine_config + self.optimizer_config = self.config.optimizer_config + self.checkpoint_config = self.config.checkpoint_config + self.device_name = get_device_name() + + if self.engine_config is None: + assert self.optimizer_config is None + if self.config.auto_select_engine_optim_fn is None: + raise ValueError( + "engine_config is not provided and auto_select_engine_optim_fn is not set. " + "Cannot determine engine backend." + ) + # Support automatically select engine backend given model config + self.engine_config, self.optimizer_config = self.config.auto_select_engine_optim_fn( + self.model_config, self.device_name + ) + + # we use the one defined in model + # TODO: this is not elegant and should refactor later + self.engine_config.use_remove_padding = self.model_config.use_remove_padding + self.engine_config.use_fused_kernels = self.model_config.use_fused_kernels + + if repatch is not None: + # NPU MindSpeed patch, will be refactored with MindSpeedEngine. + repatch(self.engine_config.get("override_transformer_config", {})) + + # TODO: add DistProfilerExtension + self.profiler_config = self.config.profiler_config + if self.profiler_config is not None: + self.profiler_tool_config = self.profiler_config.tool_config.get(self.profiler_config.tool, {}) + else: + self.profiler_tool_config = None + + DistProfilerExtension.__init__( + self, DistProfiler(rank=self.rank, config=self.profiler_config, tool_config=self.profiler_tool_config) + ) + + # self.engine: BaseEngine = EngineRegistry.new( + # model_type=self.config.model_type, + # backend=self.engine_config.strategy, + # model_config=self.model_config, + # engine_config=self.engine_config, + # optimizer_config=self.optimizer_config, + # checkpoint_config=self.checkpoint_config, + # ) + + # # build dispatch info + # self._register_dispatch_collect_info( + # mesh_name="train", + # dp_rank=self.engine.get_data_parallel_rank(), + # is_collect=self.engine.is_mp_src_rank_with_outputs(), + # ) + + self.flops_counter = FlopsCounter(self.model_config.hf_config) + + self.loss_fn = None + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def to(self, device, model=True, optimizer=True, grad=True): + """Manual control of load/offload""" + assert device in ["cpu", "device"] + + if device == "device": + device = get_device_name() + + self.engine.to(device=device, model=model, optimizer=optimizer, grad=grad) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def set_loss_fn(self, loss_fn): + self.loss_fn = loss_fn + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def reset(self): + """ + Reset the model engine to the initial state. If the engine is not initialized, + we initialize it. Otherwise, reload ckpt and reset states + """ + pass # self.engine.initialize() + + def _postprocess_output(self, output, *, global_token_num, delta_time, forward_only, images_seqlens): + """ + + Args: + output: a dictionary containing loss, model_outputs and metrics + + Returns: + + """ + # TODO: whether to log memory + # metrics["perf/max_memory_allocated_gb"] = get_torch_device().max_memory_allocated() / (1024 ** 3) + # metrics["perf/max_memory_reserved_gb"] = get_torch_device().max_memory_reserved() / (1024 ** 3) + # metrics["perf/cpu_memory_used_gb"] = psutil.virtual_memory().used / (1024 ** 3) + + metrics: dict = output.pop("metrics") + # perform all gather in dp group to ensure that it's correct. + # Here each metric in metrics can be a list (micro-batch metrics) or a singleton + # we should always sum the loss of each micro-batch as we scale by global_bsz/global_token + loss = torch.sum(torch.tensor(output.pop("loss"), device=self.device_name)) + + # For grad_norm, we do not perform all reduce because it is already been done when clipping grad + grad_norm = metrics.pop("grad_norm", None) + lr = metrics.pop("lr", None) + + final_metrics = metrics + + final_metrics["loss"] = loss + if grad_norm is not None: + final_metrics["grad_norm"] = grad_norm + if lr is not None: + final_metrics["lr"] = lr + + # TODO: confirm the mtp loss IS same across dp + for k, v in final_metrics.items(): + if k.startswith("mtp_losses"): + flatten_v = [sublist[0] for sublist in v] # sublist should be single element + final_metrics[k] = sum(flatten_v) / len(flatten_v) + # compute mfu + if global_token_num is not None: + estimated_flops, promised_flops = self.flops_counter.estimate_flops( + global_token_num, delta_time, images_seqlens=images_seqlens + ) + final_metrics["mfu"] = estimated_flops / promised_flops + if forward_only: + final_metrics["mfu"] /= 3.0 + # model outputs + model_output = output.pop("model_output", {}) + # We only return final_metrics + final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": final_metrics}) + return final_output + + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train"), blocking=False) + def train_mini_batch(self, data: TensorDict) -> TensorDict: + """Split a batch into N mini-batches run for multiple epochs + + Args: + data: + + Returns: + + """ + batch_size_per_dp = data.shape[0] + disable_auto_offload = tu.pop(data, key="disable_auto_offload", default=False) + mini_batch_size = tu.pop(data, key="mini_batch_size", default=None) + num_mini_batch = tu.pop(data, key="num_mini_batch", default=None) + epochs = tu.pop(data, key="epochs", default=1) + seed = tu.pop(data, key="seed", default=42) + dataloader_kwargs = tu.pop(data, key="dataloader_kwargs", default={}) + + self.engine_config = self.config.engine_config + + assert mini_batch_size is not None or num_mini_batch is not None + + mini_batch_size_per_gpu = mini_batch_size + + # make iterator + dataloader = tu.make_iterator( + data, + mini_batch_size=mini_batch_size_per_gpu, + epochs=epochs, + seed=seed, + dataloader_kwargs=dataloader_kwargs, + ) + + with ( + Timer(name="train_batch", logger=None), + ): + # update + output_lst = [] + total_num_iterations = data.shape[0] // mini_batch_size_per_gpu * epochs + + for batch_idx, mini_batch_td in enumerate(dataloader): + # add global token num + global_token_num = mini_batch_td["input_ids"].offsets().diff().tolist() # (total_nnz,) + tu.assign_non_tensor( + mini_batch_td, + global_token_num=NonTensorData(global_token_num), + update_lr_scheduler=batch_idx == total_num_iterations - 1, + disable_auto_offload=True, + ) + actor_output = self.train_batch(mini_batch_td) + output_lst.append(actor_output) + + actor_output = [tu.get(output, "metrics") for output in output_lst] + metrics = {} + for output in actor_output: + for key, val in output.items(): + print(f"metrics {key=} {val=}") + + # flattn dp and micro batch + if isinstance(val, list): + output[key] = ( + Metric.aggregate_dp(val) + if isinstance(val[0], Metric) + else list(chain.from_iterable(val)) + ) + append_to_dict(metrics, output) + + output = tu.get_tensordict(tensor_dict={}, non_tensor_dict={"metrics": metrics}).cpu() + + return output + + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train"), blocking=False) + def train_batch(self, data: TensorDict) -> TensorDict: + assert self.loss_fn is not None, "loss function can't be None when calling train_batch" + + # global_token_num should be a list of number of tokens of each seq in this batch + global_token_num = tu.get(data, key="global_token_num") + disable_auto_offload = tu.get(data, key="disable_auto_offload", default=False) + images_seqlens = tu.get(data, key="images_seqlens", default=None) + + # inject engineering parameters if not specified + default_keys = dict( + use_remove_padding=self.model_config.use_remove_padding, + use_dynamic_bsz=self.engine_config.use_dynamic_bsz, + max_token_len_per_gpu=self.engine_config.max_token_len_per_gpu, + micro_batch_size_per_gpu=self.engine_config.micro_batch_size_per_gpu, + use_fused_kernels=self.engine_config.use_fused_kernels, + ) + + for key, val in default_keys.items(): + if key not in data.keys(): + tu.assign_non_tensor(data, **{key: val}) + + with ( + Timer(name="train_batch", logger=None) as timer, + ): + # XXX: what's missing is the loss function to be run on the dss side + # arctic-verl/verl/workers/engine/fsdp/transformer_impl.py:1098 forward_step + # the loss function is arctic-verl/verl/workers/utils/losses.py:97 ppo_loss + # from verl.workers.utils.losses import ppo_loss <- need to adapt to pass a gazillion of config variables + + # from verl.utils.tensordict_utils import chunk_tensordict + # batch = chunk_tensordict(data, 1) + print(f"update_actor data: {data}") + + # XXX: fixme + # batch = batch[0] + + dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + print(f"{dss_batch_dict=}") + print(f"{output_args=}") + # import pdb; pdb.set_trace() + + # we need to serialize the config object to dict + # dataclasses.asdict only returns keys that are defined at init (vars will do more) - but perhaps we want `asdict`? + actor_config_as_dict = vars(self.actor_config) + print(f"update_actor: {self.actor_config=}") + print(f"update_actor: {actor_config_as_dict}") + import json + def safe_serialize(obj): + return json.loads(json.dumps(obj, default=lambda o: None)) + #actor_config_as_dict = safe_serialize(self.actor_config) + actor_config_as_dict = safe_serialize(actor_config_as_dict) + + + # pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) + # batch_size = data["input_ids"].shape[0] + # seq_len_effective = data["input_ids"].offsets().diff() + # max_seq_len = max(seq_len_effective) + # ready_input_ids = torch.nested.to_padded_tensor( + # data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) + # ) + # ready_position_ids = torch.nested.to_padded_tensor( + # data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) + # ) + # extra_inputs = dict( + # prompts=data["prompts"], + # responses=data["responses"], + # attention_mask=data["attention_mask"], + # max_response_len=data["max_response_len"], + # global_batch_size=data["global_batch_size"], + # response_mask=data["response_mask"], + # old_log_probs=data["old_log_probs"], + # advantages=data["advantages"], + # ref_log_prob=data["ref_log_prob"], + # rollout_is_weights=data.get("rollout_is_weights", None), + # ready_input_ids=ready_input_ids, + # ready_position_ids=ready_position_ids, + # ready_labels=ready_input_ids, + # # =batch[""], + # ) + # extra_inputs["batch_num_tokens"] = data["loss_mask"].sum() + + extra_inputs = prepare_extra_inputs(data) + policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) + + post_process_inputs = dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config, extra_inputs=extra_inputs) + print(f"update_actor: {post_process_inputs=}") + + # XXX: pass the original batch as post_process_inputs["batch"] - the ppo loss function expects data["prompts"] + # it got stripped and is not in dss_batch_dict +# File "/code/users/stas/github/sf/dss-platform/dss/processors/verl.py", line 90, in fwd_post_process_ppo_loss +# return ppo_loss(config, model_output, data) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "/code/users/stas/github/sf/dss-platform/dss/processors/verl.py", line 100, in ppo_loss +# log_prob = no_padding_2_padding(model_output["log_probs"], data) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "/code/users/stas/github/sf/arctic-verl/verl/workers/utils/padding.py", line 99, in no_padding_2_padding +# prompt_ids = data["prompts"] +# ~~~~^^^^^^^^^^^ +# KeyError: 'prompts' + + loss, metrics = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) + # output = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) + print(f"update_actor: {loss=}") + print(f"update_actor: {metrics=}") + + + from verl.utils.metric import AggregationType, Metric + # XXX: fix me - we need to aggregate the metrics + metrics = {k:Metric(value=v[0], aggregation=AggregationType.MEAN) for k,v in metrics.items()} + + delta_time = timer.last + + # XXX: fix me + # metrics = { + # 'actor/pg_clipfrac': None, + # 'actor/ppo_kl': None, + # 'actor/pg_clipfrac_lower': None, + # 'actor/pg_loss': None, + # 'kl_loss': None, + # 'kl_coef': None, + # 'grad_norm': None, + # } + + print(f"{data=}") + print(f"{data["input_ids"].shape=}") + model_output = { + # XXX: fix me - made a copy of existing same shape tensor for now + # 'log_probs': batch[0]["ref_log_prob"] + } + + # expected output so far + # + # output={ + # 'model_output': { + # 'log_probs': NestedTensor(size=(1,j18), offsets=tensor([ 0,401], device='cuda:0'), grad_fn=, contiguous=True) + # }, + # 'loss': [-0.9999991059303284], + # 'metrics': { + # 'actor/pg_clipfrac': , + # 'actor/ppo_kl': , + # 'actor/pg_clipfrac_lower': , + # 'actor/pg_loss': , + # 'kl_loss': , + # 'kl_coef': [0.001], + # 'grad_norm': 16.321151733398438, + # } + # } + + + #output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict=non_tensor_dict) + output = dict( + model_output=model_output, + metrics=metrics, + loss=loss, + ) + + update_lr_scheduler = tu.get(data, key="update_lr_scheduler", default=False) + # XXX: fix me + update_lr_scheduler = False + # update lr scheduler + if update_lr_scheduler: + lr = self.engine.lr_scheduler_step() + else: + lr = None + + + # we don't need model_output in training. Maybe we change out mind later + #output.pop("model_output") + if lr is not None: + output["metrics"]["lr"] = lr + + final_output = self._postprocess_output( + output, + global_token_num=global_token_num, + delta_time=delta_time, + forward_only=False, + images_seqlens=images_seqlens, + ).cpu() + + return final_output + + + + + + +class ActorRolloutRefWorker(Worker, DistProfilerExtension): + def __init__(self, config: DictConfig, role: str, **kwargs): + Worker.__init__(self) + self.config = config + self.role = role + self._is_actor = self.role in ["actor", "actor_rollout", "actor_rollout_ref"] + self._is_rollout = self.role in ["rollout", "actor_rollout", "actor_rollout_ref"] + self._is_ref = self.role in ["ref", "actor_rollout_ref"] + + self.arctic_rl_client = kwargs.get("arctic_rl_client", None) + + # assert self.arctic_rl_client is not None, "arctic_rl_client is required" + self._loaded_dump_data = load_dump_data(1, 1) + DistProfilerExtension.__init__(self, DistProfiler(rank=self.rank, config=None, tool_config=None)) + + if self._is_actor: + model_config: HFModelConfig = omega_conf_to_dataclass(self.config.model) + actor_config: ActorConfig = omega_conf_to_dataclass(self.config.actor) + actor_config.model_config = model_config + actor_training_config = TrainingWorkerConfig( + model_type="language_model", + model_config=actor_config.model_config, + engine_config=actor_config.engine, + optimizer_config=actor_config.optim, + checkpoint_config=actor_config.checkpoint, + ) + self.actor_config = actor_config + + assert self.config.actor.use_dynamic_bsz == self.config.rollout.log_prob_use_dynamic_bsz + + # assign engine configs + actor_training_config.engine_config.use_dynamic_bsz = self.config.actor.use_dynamic_bsz + actor_training_config.engine_config.infer_max_token_len_per_gpu = ( + self.config.rollout.log_prob_max_token_len_per_gpu + ) + actor_training_config.engine_config.infer_micro_batch_size_per_gpu = ( + self.config.rollout.log_prob_micro_batch_size_per_gpu + ) + actor_training_config.engine_config.max_token_len_per_gpu = self.config.actor.ppo_max_token_len_per_gpu + actor_training_config.engine_config.micro_batch_size_per_gpu = ( + self.config.actor.ppo_micro_batch_size_per_gpu + ) + actor_training_config.engine_config.use_remove_padding = model_config.use_remove_padding + + if self.config.actor.use_dynamic_bsz: + assert self.config.rollout.log_prob_max_token_len_per_gpu is not None + assert self.config.actor.ppo_max_token_len_per_gpu is not None + else: + assert self.config.rollout.log_prob_micro_batch_size_per_gpu is not None + assert self.config.actor.ppo_micro_batch_size_per_gpu is not None + + self.actor = TrainingWorker(config=actor_training_config, actor_config=actor_config, arctic_rl_client=self.arctic_rl_client) + + self.actor.reset() + self.loss_fn = partial(ppo_loss, config=actor_config) + self.actor.set_loss_fn(loss_fn=self.loss_fn) + + self.set_dispatch_collect(mesh_name="actor", **self.actor.get_dispatch_collect()) + + # from verl.workers.actor import DataParallelPPOActor + + # # hacks to appease to DataParallelPPOActor + # import torch.distributed + # torch.distributed.get_rank = lambda: 0 + + # actor_cfg = omega_conf_to_dataclass(self.config.actor) + # self.actor = DataParallelPPOActor( + # # XXX: hijack actor_module + # config=actor_cfg, actor_module=None, actor_optimizer=None + # ) + + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + self._register_dispatch_collect_info("actor", dp_rank=self.rank, is_collect=True) + self._register_dispatch_collect_info("ref", dp_rank=self.rank, is_collect=True) + self._register_dispatch_collect_info("rollout", dp_rank=self.rank, is_collect=True) + + return + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def destroy(self): + self.dss_training_engine.destroy() + self.arctic_inference_engine.destroy() + return + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def set_loss_fn(self, loss_fn): + return + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def to(self, device, model=True, optimizer=True, grad=True): + """Manual control of load/offload""" + return + + + def _update_config_params(self, data: TensorDict): + default_keys = dict( + use_remove_padding=self.actor.model_config.use_remove_padding, + use_dynamic_bsz=self.actor.engine_config.use_dynamic_bsz, + max_token_len_per_gpu=self.actor.engine_config.max_token_len_per_gpu, + micro_batch_size_per_gpu=self.actor.engine_config.micro_batch_size_per_gpu, + use_fused_kernels=self.actor.engine_config.use_fused_kernels, + ) + + for key, val in default_keys.items(): + if key not in data.keys(): + tu.assign_non_tensor(data, **{key: val}) + + + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="ref")) + @DistProfiler.annotate(color="olive", role="ref_compute_log_prob") + def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: + # return self._loaded_dump_data["full_ref_log_prob"] + # import pdb; pdb.set_trace() + # from verl.utils.tensordict_utils import chunk_tensordict + # batch = chunk_tensordict(data, 1) + # print(f"compute_ref_log_prob data: {data}") + dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + # print(f"{dss_batch_dict=}") + # import pdb; pdb.set_trace() + # self.dss_training_engine.forward(**dss_batch_dict) + # loss = self.dss_training_engine.backward() + # print(f"loss: {loss}") + # import pdb; pdb.set_trace() + # log_prob = self._loaded_dump_data["full_log_prob"] + + self._update_config_params(data) + post_process_inputs = prepare_log_prob_extra_inputs(data) + entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) + + batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) + model_output = batch_output.pop("model_output", {}) + metrics = { + "mfu": 0.0, + "loss": 1.0, + } + final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) + + + # metrics = { + # "mfu": 0.0, + # "loss": 1.0, + # "batch_size": 1, + # } + + # model_output = { + # "log_probs": log_probs, + # } + + # final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) + + return final_output + + + + + # TODO: Actor API Begin + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") + def compute_log_prob(self, data: TensorDict) -> TensorDict: + # import pdb; pdb.set_trace() + # from verl.utils.tensordict_utils import chunk_tensordict + # batch = chunk_tensordict(data, 1) + # print(f"compute_log_prob data: {data}") + dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + # print(f"{dss_batch_dict=}") + # import pdb; pdb.set_trace() + # self.dss_training_engine.forward(**dss_batch_dict) + # loss = self.dss_training_engine.backward() + # print(f"loss: {loss}") + # import pdb; pdb.set_trace() + # log_prob = self._loaded_dump_data["full_log_prob"] + + self._update_config_params(data) + post_process_inputs = prepare_log_prob_extra_inputs(data) + # print(f"compute_log_prob: {post_process_inputs=}") + + entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) + + # import pdb; pdb.set_trace() + batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) + model_output = batch_output.pop("model_output", {}) + metrics = { + "mfu": 0.0, + "loss": 1.0, + } + final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) + + return final_output + + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + @DistProfiler.annotate(color="red", role="actor_update") + def update_actor(self, data: TensorDict) -> TensorDict: + output = self.actor.train_mini_batch(data=data) + return output.cpu() if output is not None else None + + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def load_checkpoint(self, local_path, hdfs_path=None, del_local_after_load=False): + assert "actor" in self.role, "load_checkpoint only support actor role" + return + + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): + assert "actor" in self.role, "save_checkpoint only support actor role" + return + + @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) + async def update_weights(self, global_steps: int = None): + """Update weights from trainer to rollout. + + 1. For sync training with colocated trainer and rollout, update rollout directly from model engine. + - before update_weights: rollout should be in sleep mode. + - after update_weights: rollout should be in wake_up mode. + 2. For async training with disaggregated trainer and rollout, send_weights only by checkpoint engine. + """ + return + + # TODO: Actor API End + + + # TODO: Rollout API Begin + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="rollout")) + async def generate_sequences(self, batch: DataProto): + print(f"{batch.non_tensor_batch=}") + raw_prompts = list(batch.non_tensor_batch["raw_prompt"]) + print(f"{raw_prompts=}") + prompts = self.tokenizer.apply_chat_template( + raw_prompts, + add_generation_prompt=True, + tokenize=False, + ) + # import pdb; pdb.set_trace() + print(f"prompts: {prompts}") + metrics = {} + + gen_batch_output = self.arctic_inference_engine.generate(prompts=prompts) + + return gen_batch_output + # return self._loaded_dump_data["gen_batch_output"] + + # TODO: Rollout API End + + # TODO: CheckpointManager API Begin + @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) + async def sleep_replicas(self): + """Sleep all rollout replicas: free weight and kv_cache device memory.""" + return + # TODO: CheckpointManager API \ No newline at end of file diff --git a/verl/workers/rollout/arctic_rollout/__init__.py b/verl/workers/rollout/arctic_rollout/__init__.py new file mode 100644 index 00000000000..cf453f8b2f6 --- /dev/null +++ b/verl/workers/rollout/arctic_rollout/__init__.py @@ -0,0 +1,3 @@ +from .arctic_rollout import ArcticReplica + +__all__ = ["ArcticReplica"] diff --git a/verl/workers/rollout/arctic_rollout/arctic_rollout.py b/verl/workers/rollout/arctic_rollout/arctic_rollout.py new file mode 100644 index 00000000000..c744b4a00fd --- /dev/null +++ b/verl/workers/rollout/arctic_rollout/arctic_rollout.py @@ -0,0 +1,322 @@ +import ray +from typing import Any, Optional +from verl.workers.rollout.vllm_rollout.vllm_async_server import vLLMHttpServer + +import argparse +from typing import Any, Optional +from verl.trainer.ppo.arctic_rl_client import ArcticRLClient4VeRL +from collections.abc import AsyncGenerator + +import ray +from ray.actor import ActorHandle +from vllm import SamplingParams +from vllm.inputs import TokensPrompt +from vllm.lora.request import LoRARequest +from vllm.outputs import RequestOutput, CompletionOutput + +from verl.utils.tokenizer import normalize_token_ids +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.replica import RolloutMode, RolloutReplica, TokenOutput +from verl.workers.rollout.vllm_rollout.utils import ( + VLLM_LORA_INT_ID, + VLLM_LORA_NAME, + VLLM_LORA_PATH, +) +from transformers import AutoTokenizer +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.rollout.utils import get_max_position_embeddings + + +class ArcticLLMEngine: + def __init__( + self, + replica_rank: int, + arctic_rl_client: ArcticRLClient4VeRL, + ): + self.replica_rank = replica_rank + self.arctic_rl_client = arctic_rl_client + self.tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") + + async def generate( + self, + prompt: TokensPrompt, + sampling_params: dict[str, Any], + request_id: str, + lora_request: Optional[LoRARequest] = None, + priority: int = 0, + ) -> AsyncGenerator[RequestOutput, None]: + gen_batch_output = await self.arctic_rl_client.generate.remote( + prompt_ids=prompt['prompt_token_ids'], + sampling_params=sampling_params, + ) + # print(f"arctic_async_server: {gen_batch_output=}, {type(gen_batch_output)=}") + # gen_batch_output = await ray.get(gen_batch_output) + + raw_prompt = self.tokenizer.decode(prompt['prompt_token_ids']) + completed_outputs = [] + for i, output in enumerate(gen_batch_output): + completed_outputs.append(CompletionOutput( + index=i, + text=output['text'], + token_ids=output['token_ids'], + finish_reason=output['finish_reason'], + cumulative_logprob=None, + logprobs=None + ) + ) + + yield RequestOutput( + request_id=request_id, + outputs=completed_outputs, + prompt=raw_prompt, + prompt_logprobs=None, + prompt_token_ids=prompt['prompt_token_ids'], + # finished=completed_output.finish_reason == "stop", + finished=True, + ) + + +class ArcticLLMServer(vLLMHttpServer): + """vLLM http server in single node, this is equivalent to launch server with command line: + ``` + vllm serve --tensor-parallel-size=8 ... + ``` + """ + + def __init__( + self, + config: RolloutConfig, + model_config: HFModelConfig, + rollout_mode: RolloutMode, + arctic_rl_client: ArcticRLClient4VeRL, + workers: list[ActorHandle] = [], + replica_rank: int = 0, + node_rank: int = 0, + gpus_per_node: int = 1, + nnodes: int = 1, + cuda_visible_devices: str = "0", + ): + """ + Args: + config (RolloutConfig): full config. + model_config (HFModelConfig): model config. + rollout_mode (RolloutMode): rollout mode. + replica_rank (int): replica rank, a replica may contain multiple nodes. + node_rank (int): node rank. + gpus_per_node (int): number of gpus per node. + nnodes (int): number of nodes. + cuda_visible_devices (str): cuda visible devices. + """ + self.config: RolloutConfig = omega_conf_to_dataclass(config) + self.model_config: HFModelConfig = omega_conf_to_dataclass(model_config, dataclass_type=HFModelConfig) + max_position_embeddings = get_max_position_embeddings(self.model_config.hf_config) + if self.config.max_model_len is None: + self.config.max_model_len = max_position_embeddings + else: + if self.config.max_model_len > max_position_embeddings: + raise ValueError( + f"max_model_len ({self.config.max_model_len}) should be less than or equal to " + f"max_position_embeddings ({max_position_embeddings})" + ) + + self.rollout_mode = rollout_mode + self.workers = workers + + self.replica_rank = replica_rank + self.node_rank = node_rank + self.gpus_per_node = gpus_per_node + self.nnodes = nnodes + # model weights version, set by ServerAdapter when update weights. + self.global_steps = None + + if self.rollout_mode != RolloutMode.HYBRID and self.config.load_format == "dummy": + # logger.warning(f"rollout mode is {self.rollout_mode}, load_format is dummy, set to auto") + self.config.load_format = "auto" + + + self._master_address = None + self._master_port = None + self._dp_rpc_port = None + self._dp_master_port = None + + self.engine = ArcticLLMEngine(replica_rank, arctic_rl_client) + + # logger.info( + # f"vLLMHttpServer, replica_rank: {self.replica_rank}, node_rank: {self.node_rank}, " + # f"{get_visible_devices_keyword()}: {cuda_visible_devices}, " + # f"master_address: {self._master_address}, master_port: {self._master_port}, " + # f"data_parallel_rpc_port: {self._dp_rpc_port}, data_parallel_master_port: {self._dp_master_port}" + # ) + + def get_master_address(self): pass + + def get_server_address(self): pass + + @property + def lora_as_adapter(self) -> bool: pass + + async def collective_rpc( + self, + **kwargs, + ): + pass + + async def launch_server(self, master_address: str = None, master_port: int = None, dp_rpc_port: int = None): + pass + + async def run_server(self, args: argparse.Namespace): + pass + + + async def generate( + self, + prompt_ids: list[int], + sampling_params: dict[str, Any], + request_id: str, + image_data: Optional[list[Any]] = None, + video_data: Optional[list[Any]] = None, + priority: int = 0, + ) -> TokenOutput: + """Generate sequence with token-in-token-out.""" + prompt_ids = normalize_token_ids(prompt_ids) + + # Calculate the maximum possible new tokens based on available context space + # This serves as a safety upper bound + max_possible_tokens = self.config.max_model_len - len(prompt_ids) + if max_possible_tokens < 0: + raise ValueError( + f"Prompt length ({len(prompt_ids)}) exceeds the model's maximum context length " + f"({self.config.max_model_len})." + ) + + # Determine max_tokens from sampling_params or use configured response_length as default + if "max_tokens" in sampling_params: + max_tokens = sampling_params.pop("max_tokens") + elif "max_new_tokens" in sampling_params: + # support sglang-style 'max_new_tokens' param + max_tokens = sampling_params.pop("max_new_tokens") + else: + # Default to a calculation that considers configured lengths + max_tokens = self.config.response_length + self.config.prompt_length - len(prompt_ids) + + # Clamp max_tokens to the valid range [0, max_possible_tokens] + max_tokens = max(0, min(max_tokens, max_possible_tokens)) + + assert max_tokens <= max_possible_tokens, ( + f"max_tokens {max_tokens} exceeds available context space {max_possible_tokens}" + ) + sampling_params["logprobs"] = 0 if sampling_params.pop("logprobs", False) else None + sampling_params.setdefault("repetition_penalty", self.config.get("repetition_penalty", 1.0)) + # sampling_params = SamplingParams(max_tokens=max_tokens, **sampling_params) + sampling_params["max_tokens"] = max_tokens + multi_modal_data = {} + if image_data is not None: + multi_modal_data["image"] = image_data + if video_data is not None: + multi_modal_data["video"] = video_data + # import pdb; pdb.set_trace() + prompt = TokensPrompt(prompt_token_ids=prompt_ids, multi_modal_data=multi_modal_data) + + # Add lora request + lora_request = None + if self.lora_as_adapter: + # Make sure we also check that the lora is already loaded in the engine + lora_loaded = VLLM_LORA_INT_ID in await self.engine.list_loras() + if lora_loaded: + lora_request = LoRARequest( + lora_name=VLLM_LORA_NAME, lora_int_id=VLLM_LORA_INT_ID, lora_path=VLLM_LORA_PATH + ) + # import pdb; pdb.set_trace() + generator = self.engine.generate( + prompt=prompt, + sampling_params=sampling_params, + request_id=request_id, + lora_request=lora_request, + priority=priority, + ) + + # print(f"arctic_async_server: {generator=}, {type(generator)=}") + + # Get final response + final_res: Optional[RequestOutput] = None + async for output in generator: + final_res = output + assert final_res is not None + + token_ids = final_res.outputs[0].token_ids + log_probs = None + if sampling_params["logprobs"] is not None: + log_probs = [logprobs[token_ids[i]].logprob for i, logprobs in enumerate(final_res.outputs[0].logprobs)] + + routed_experts = None + if self.config.enable_rollout_routing_replay: + routed_experts = final_res.outputs[0].routed_experts + + # Determine stop reason from finish_reason + finish_reason = final_res.outputs[0].finish_reason + if finish_reason == "abort": + stop_reason = "aborted" + elif finish_reason in ("stop", "length"): + stop_reason = "completed" + else: + stop_reason = finish_reason # for more stop reason in the future + + num_preempted = None + + if hasattr(final_res.outputs[0], "num_preempted"): + num_preempted = final_res.outputs[0].num_preempted + + return TokenOutput( + token_ids=token_ids, + log_probs=log_probs, + routed_experts=routed_experts, + stop_reason=stop_reason, + num_preempted=num_preempted, + extra_info={"global_steps": self.global_steps}, + ) + + + + +class ArcticReplica(RolloutReplica): + def __init__( + self, + replica_rank: int, + config: RolloutConfig, + model_config: HFModelConfig, + gpus_per_node: int = 1, + is_reward_model: bool = False, + **kwargs, + ): + super().__init__(replica_rank, config, model_config, gpus_per_node, is_reward_model) + self.server_class = ray.remote(ArcticLLMServer) + self.arctic_rl_client = kwargs.get("arctic_rl_client", None) + # assert self.arctic_rl_client is not None, "arctic_rl_client is required" + + + def rollout_worker_use_gpu(self) -> bool: + return False + + + async def launch_servers(self): + server = self.server_class.options( + ).remote( + replica_rank=self.replica_rank, + config=self.config, + model_config=self.model_config, + rollout_mode=self.rollout_mode, + arctic_rl_client=self.arctic_rl_client, + ) + self.servers.append(server) + self._server_handle = server + + + async def wake_up(self): + pass + + async def sleep(self): + pass + + async def abort_request(self, request_id: str) -> dict[str, Any]: + return {"aborted": True, "request_id": 0} + \ No newline at end of file From cadd1ee231afedef6baa510eeaa0e6e614ec1d7a Mon Sep 17 00:00:00 2001 From: Tunji Ruwase Date: Thu, 2 Apr 2026 13:29:08 -0400 Subject: [PATCH 03/58] Revert changes (#7) --- examples/arctic_rl/run_arctic_gsm8k_grpo.sh | 86 -- examples/arctic_rl/run_gsm8k_grpo.sh | 81 -- verl/experimental/agent_loop/agent_loop.py | 9 +- verl/single_controller/ray/base.py | 15 +- verl/trainer/config/ppo_trainer.yaml | 3 - verl/trainer/main_ppo.py | 15 +- verl/trainer/ppo/arctic_rl_client.py | 130 --- verl/trainer/ppo/arctic_trainer.py | 153 --- verl/trainer/ppo/ray_trainer.py | 10 +- verl/utils/fsdp_utils.py | 13 - verl/workers/arctic_workers.py | 918 ------------------ verl/workers/engine_workers.py | 2 +- .../rollout/arctic_rollout/__init__.py | 3 - .../rollout/arctic_rollout/arctic_rollout.py | 322 ------ verl/workers/rollout/replica.py | 5 - 15 files changed, 15 insertions(+), 1750 deletions(-) delete mode 100755 examples/arctic_rl/run_arctic_gsm8k_grpo.sh delete mode 100755 examples/arctic_rl/run_gsm8k_grpo.sh delete mode 100644 verl/trainer/ppo/arctic_rl_client.py delete mode 100644 verl/trainer/ppo/arctic_trainer.py delete mode 100644 verl/workers/arctic_workers.py delete mode 100644 verl/workers/rollout/arctic_rollout/__init__.py delete mode 100644 verl/workers/rollout/arctic_rollout/arctic_rollout.py diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh deleted file mode 100755 index 44cc760c708..00000000000 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -set -x - -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -# we want to make sure this runs on non-gpu client -export CUDA_VISIBLE_DEVICES= - -# BSZ=1024 -BSZ=2 -MBS=2 -UBS=2 -ROLL_N=2 -MAX_STEPS=1 -# LR=0 -LR=1e-6 -LOGGER=console -# LOGGER="['console','wandb']" -USE_KL_LOSS=True -# USE_KL_LOSS=False -# REMOVE_PADDING=True -REMOVE_PADDING=False -MODEL="Qwen/Qwen3-0.6B" -# STRATEGY="fsdp" -STRATEGY="fsdp2" -PYTHONUNBUFFERED=1 -HYDRA_FULL_ERROR=1 -USE_LEGACY_WORKER_IMPL=disable -NGPU_PER_NODE=1 -ROLLOUT_NAME=arctic # entry point into ArcticRL -USE_ARCTIC_RL=True - -experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_at${USE_ARCTIC_RL}" - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=/code/shared/gsm8k/train.parquet \ - data.val_files=/code/shared/gsm8k/test.parquet \ - data.train_batch_size=${BSZ} \ - data.max_prompt_length=64 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.shuffle=False \ - reward.num_workers=1 \ - actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.model.path=${MODEL} \ - actor_rollout_ref.actor.optim.lr=${LR} \ - actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ - actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.strategy=${STRATEGY} \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.enforce_eager=True \ - actor_rollout_ref.rollout.n=${ROLL_N} \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.ref.fsdp_config.param_offload=False \ - actor_rollout_ref.ref.strategy=${STRATEGY} \ - algorithm.use_kl_in_reward=False \ - trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ - trainer.use_arctic_rl=${USE_ARCTIC_RL} \ - trainer.critic_warmup=0 \ - trainer.logger=${LOGGER} \ - trainer.experiment_name=${experiment_name} \ - trainer.project_name='verl_arctic_grpo_gsm8k' \ - trainer.val_before_train=False \ - trainer.n_gpus_per_node=${NGPU_PER_NODE} \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_training_steps=${MAX_STEPS} \ - trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log - - # trainer.total_training_steps=${MAX_STEPS} \ diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh deleted file mode 100755 index f65b0f55b24..00000000000 --- a/examples/arctic_rl/run_gsm8k_grpo.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -set -x - -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -# BSZ=1024 -BSZ=2 -MBS=2 -UBS=2 -ROLL_N=2 -MAX_STEPS=1 -# LR=0 -LR=1e-6 -LOGGER=console -# LOGGER="['console','wandb']" -USE_KL_LOSS=True -# USE_KL_LOSS=False -# REMOVE_PADDING=True -REMOVE_PADDING=False -MODEL="Qwen/Qwen3-0.6B" -# STRATEGY="fsdp" -STRATEGY="fsdp2" -PYTHONUNBUFFERED=1 -HYDRA_FULL_ERROR=1 -USE_LEGACY_WORKER_IMPL=disable -NGPU_PER_NODE=1 -ROLLOUT_NAME=vllm - -experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}" - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=/code/shared/gsm8k/train.parquet \ - data.val_files=/code/shared/gsm8k/test.parquet \ - data.train_batch_size=${BSZ} \ - data.max_prompt_length=64 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.shuffle=False \ - reward.num_workers=1 \ - actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.model.path=${MODEL} \ - actor_rollout_ref.actor.optim.lr=${LR} \ - actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ - actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.strategy=${STRATEGY} \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.enforce_eager=True \ - actor_rollout_ref.rollout.n=${ROLL_N} \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.ref.fsdp_config.param_offload=False \ - actor_rollout_ref.ref.strategy=${STRATEGY} \ - algorithm.use_kl_in_reward=False \ - trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ - trainer.critic_warmup=0 \ - trainer.logger=${LOGGER} \ - trainer.experiment_name=${experiment_name} \ - trainer.project_name='verl_arctic_grpo_gsm8k' \ - trainer.val_before_train=False \ - trainer.n_gpus_per_node=${NGPU_PER_NODE} \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_training_steps=${MAX_STEPS} \ - trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log - - # trainer.total_training_steps=${MAX_STEPS} \ diff --git a/verl/experimental/agent_loop/agent_loop.py b/verl/experimental/agent_loop/agent_loop.py index 8e7118db65e..8879960f128 100644 --- a/verl/experimental/agent_loop/agent_loop.py +++ b/verl/experimental/agent_loop/agent_loop.py @@ -918,14 +918,13 @@ def __init__( worker_group: RayWorkerGroup = None, rollout_resource_pool: RayResourcePool = None, reward_loop_worker_handles: list[ray.actor.ActorHandle] = None, - **kwargs, - ): + ): self.config = config self.rollout_config, self.model_config = _get_rollout_and_model_config(config) self.worker_group = worker_group self.rollout_resource_pool = rollout_resource_pool self.reward_loop_worker_handles = reward_loop_worker_handles - self.kwargs = kwargs + assert worker_group is not None or self.rollout_config.nnodes > 0, "nnodes must be > 0 in standalone mode" # for recipe to change @@ -942,10 +941,9 @@ async def create( worker_group: RayWorkerGroup = None, rollout_resource_pool: RayResourcePool = None, reward_loop_worker_handles: list[ray.actor.ActorHandle] = None, - **kwargs, ): """Create agent loop manager.""" - instance = cls(config, worker_group, rollout_resource_pool, reward_loop_worker_handles, **kwargs) + instance = cls(config, worker_group, rollout_resource_pool, reward_loop_worker_handles) await instance._initialize_llm_servers() await instance._init_global_load_balancer() await instance._init_agent_loop_workers() @@ -970,7 +968,6 @@ async def _initialize_llm_servers(self): config=self.rollout_config, model_config=self.model_config, gpus_per_node=self.rollout_config.n_gpus_per_node, - **self.kwargs, ) for replica_rank in range(num_replicas) ] diff --git a/verl/single_controller/ray/base.py b/verl/single_controller/ray/base.py index a7872b189a1..2f6ee47064f 100644 --- a/verl/single_controller/ray/base.py +++ b/verl/single_controller/ray/base.py @@ -187,9 +187,8 @@ class ResourcePoolManager: resource_pool_spec: dict[str, list[int]] mapping: dict[int, str] resource_pool_dict: dict[str, RayResourcePool] = field(default_factory=dict) - gpu_resource_pool_dict: dict[str, RayResourcePool] = field(default_factory=dict) - def create_resource_pool(self, use_gpu: bool = True): + def create_resource_pool(self): """Create Ray resource pools for distributed training. Initializes resource pools based on the resource pool specification, @@ -203,11 +202,10 @@ def create_resource_pool(self, use_gpu: bool = True): # For Megatron backend, we recommend using max_colocate_count>1 # that can utilize different WorkerGroup for differnt models resource_pool = RayResourcePool( - process_on_nodes=process_on_nodes, use_gpu=use_gpu, max_colocate_count=3, name_prefix=resource_pool_name + process_on_nodes=process_on_nodes, use_gpu=True, max_colocate_count=3, name_prefix=resource_pool_name ) self.resource_pool_dict[resource_pool_name] = resource_pool - if use_gpu: - self.gpu_resource_pool_dict[resource_pool_name] = resource_pool + self._check_resource_available() def get_resource_pool(self, role) -> RayResourcePool: @@ -216,8 +214,7 @@ def get_resource_pool(self, role) -> RayResourcePool: def get_n_gpus(self) -> int: """Get the number of gpus in this cluster.""" - process_on_gpu_nodes = [process_on_nodes for pool_name, process_on_nodes in self.resource_pool_spec.items() if pool_name in self.gpu_resource_pool_dict] - return sum([n_gpus for process_on_nodes in process_on_gpu_nodes for n_gpus in process_on_nodes]) + return sum([n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes]) def _check_resource_available(self): """Check if the resource pool can be satisfied in this ray cluster.""" @@ -229,7 +226,9 @@ def _check_resource_available(self): # check total required gpus can be satisfied total_available_gpus = sum(node_available_gpus.values()) - total_required_gpus = self.get_n_gpus() + total_required_gpus = sum( + [n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes] + ) if total_available_gpus < total_required_gpus: raise ValueError( f"Total available GPUs {total_available_gpus} is less than total desired GPUs {total_required_gpus}" diff --git a/verl/trainer/config/ppo_trainer.yaml b/verl/trainer/config/ppo_trainer.yaml index 2a0779cff1c..fd9b59862ae 100644 --- a/verl/trainer/config/ppo_trainer.yaml +++ b/verl/trainer/config/ppo_trainer.yaml @@ -203,9 +203,6 @@ trainer: # mode: "auto", "enable", or "disable" use_legacy_worker_impl: auto - # whether to use arctic rl - use_arctic_rl: False - # profiler configs global_profiler: diff --git a/verl/trainer/main_ppo.py b/verl/trainer/main_ppo.py index 262a318be8b..2c84374d245 100644 --- a/verl/trainer/main_ppo.py +++ b/verl/trainer/main_ppo.py @@ -134,10 +134,6 @@ def add_actor_rollout_worker(self, config): actor_rollout_cls = ActorRolloutRefWorker ray_worker_group_cls = RayWorkerGroup - if config.trainer.get("use_arctic_rl", False): - from verl.workers.arctic_workers import ActorRolloutRefWorker - actor_rollout_cls = ActorRolloutRefWorker - lora_rank = config.actor_rollout_ref.model.get("lora", {}).get("rank", 0) if lora_rank <= 0: lora_rank = config.actor_rollout_ref.model.get("lora_rank", 0) @@ -344,9 +340,7 @@ def run(self, config): train_sampler = create_rl_sampler(config.data, train_dataset) # Initialize the PPO trainer. - from verl.trainer.ppo.arctic_trainer import ArcticPPOTrainer - ppo_trainer_cls = RayPPOTrainer if not config.trainer.use_arctic_rl else ArcticPPOTrainer - trainer = ppo_trainer_cls( + trainer = RayPPOTrainer( config=config, tokenizer=tokenizer, processor=processor, @@ -362,12 +356,7 @@ def run(self, config): trainer.init_workers() # Start the training process. - try: - trainer.fit() - finally: - # Ensure remote services shutdown gracefully - if hasattr(trainer, "destroy"): - trainer.destroy() + trainer.fit() def create_rl_dataset(data_paths, data_config, tokenizer, processor, is_train=True, max_samples: int = -1): diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py deleted file mode 100644 index 8f7e74b21c9..00000000000 --- a/verl/trainer/ppo/arctic_rl_client.py +++ /dev/null @@ -1,130 +0,0 @@ -import torch -from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer -from deepspeed.utils import OnDevice -from dss_client.client import DSSInferenceClient, DSSTrainingClient, DSSLogProbClient -import ray -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy -from ray.util.placement_group import placement_group -from verl.workers.rollout.replica import TokenOutput -from tensordict import TensorDict -from typing import Any -from verl.utils.ray_utils import auto_await - -def create_arctic_rl_client(): - sched_pg = placement_group([{"GPU": 0, "CPU": 1}]) - arctic_rl_client = ray.remote( - num_cpus=0, - num_gpus=0, - scheduling_strategy=PlacementGroupSchedulingStrategy( - placement_group=sched_pg, - placement_group_capture_child_tasks=True, - ), - )(ArcticRLClient4VeRL).remote( - ) - - return arctic_rl_client - -def create_meta_model(name_or_path: str): - model_config = AutoConfig.from_pretrained(name_or_path) - with OnDevice(dtype=torch.float16, device='meta'): - meta_model = AutoModelForCausalLM.from_config(model_config) - return meta_model - -class ArcticRLClient4VeRL: - def __init__(self): - self.arctic_inference_client = DSSInferenceClient(dss_server_url="http://localhost:7000") - self.arctic_training_client = DSSTrainingClient(dss_server_url="http://localhost:7000") - self.arctic_log_prob_client = DSSLogProbClient(dss_server_url="http://localhost:7000") - - def initialize(self, model_name: str): - vllm_config = { - "temperature": 0.0, - "top_p": 1.0, - "top_k": 0, - "max_tokens": 1024, - "stop_sequences": [], - "stop_token_ids": [], - } - self.inference_engine = self.arctic_inference_client.initialize( - model_name=model_name, - vllm_config=vllm_config, - ) - self.log_prob_engine = self.arctic_log_prob_client.initialize( - model_name=model_name, - vllm_config=vllm_config, - ) - - ds_config = { - "train_micro_batch_size_per_gpu": 1, - "train_batch_size": 1, - "gradient_accumulation_steps": 1, - "sequence_parallel_size": 1, - "zero_optimization": { - "stage": 1, - }, - } - training_config = { - "optimizer": { - "lr": 0.0002, - "weight_decay": 0.0, - "betas": [0.9, 0.999], - }, - "lr_scheduler": {"warmup_ratio": 0.05}, - "training_horizon": 10, - "max_length": 8096, - "model_config": None, - "attn_implementation": "eager", - } - - self.training_engine = self.arctic_training_client.initialize( - model=create_meta_model(model_name), - ds_config=ds_config, - training_config=training_config) - - self.tokenizer = AutoTokenizer.from_pretrained(model_name) - - def generate(self, prompt_ids, sampling_params) -> TokenOutput: - prompts = [self.tokenizer.decode(prompt_ids)] - return self.inference_engine.generate( - prompts=prompts, - sampling_params=sampling_params, - ) - - def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict): - dss_batch_dict.update(post_process_inputs=post_process_inputs) - - # XXX: somehow we need to differentiate which model is this called on ref vs actor - at the moment it's always actor hardcoded - entropy, log_probs = self.training_engine.fwd_no_grad(**dss_batch_dict) - - # XXX: for some reason no_padding_2_padding expects a 1D tensor - not sure how it'll work for - # bs>1 - # I think it may have to do with tensor.is_nested - different path/logic - # so most likely we need to convert these 2 into TensorDict - if entropy is not None: - # prior_entropy_shape = entropy.shape - entropy = torch.tensor(entropy).squeeze() - if log_probs is not None: - # prior_log_probs_shape = log_probs.shape - log_probs = torch.tensor(log_probs).squeeze() - print(f"arctic_rl_client.compute_log_prob: {entropy.shape=}, {log_probs.shape=}") - return entropy, log_probs - - - def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): - - dss_batch_dict.update(post_process_inputs=post_process_inputs) - - #_ = self.training_engine.forward(**dss_batch_dict, post_process_inputs=post_process_inputs) - _ = self.training_engine.forward(**dss_batch_dict) - loss, metrics = self.training_engine.backward() - self.training_engine.step() - - print(f"arctic_rl_client.update_actor: {loss=}") - print(f"arctic_rl_client.update_actor: {metrics=}") - return loss.cpu().item(), metrics - - def destroy(self): - self.training_engine.destroy() - self.inference_engine.destroy() - return - diff --git a/verl/trainer/ppo/arctic_trainer.py b/verl/trainer/ppo/arctic_trainer.py deleted file mode 100644 index 2d2a4e95f7b..00000000000 --- a/verl/trainer/ppo/arctic_trainer.py +++ /dev/null @@ -1,153 +0,0 @@ -import torch -from typing import Optional -from torch.utils.data import Dataset, Sampler -from verl.trainer.ppo.ray_trainer import RayPPOTrainer -from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup, ResourcePoolManager -from verl.workers.arctic_workers import ActorRolloutRefWorker -from verl.trainer.ppo.utils import Role, WorkerType -from omegaconf import OmegaConf -from verl.single_controller.ray.base import create_colocated_worker_cls -from verl.trainer.ppo.arctic_rl_client import create_arctic_rl_client - -def my_pdb(): - return - import pdb; pdb.set_trace() - -class ArcticPPOTrainer(RayPPOTrainer): - def __init__( - self, - config, - tokenizer, - role_worker_mapping: dict[Role, WorkerType], - resource_pool_manager: ResourcePoolManager, - ray_worker_group_cls: type[RayWorkerGroup] = RayWorkerGroup, - processor=None, - train_dataset: Optional[Dataset] = None, - val_dataset: Optional[Dataset] = None, - collate_fn=None, - train_sampler: Optional[Sampler] = None, - device_name=None, - ): - super().__init__(config=config, - tokenizer=tokenizer, - processor=processor, - role_worker_mapping=role_worker_mapping, - resource_pool_manager=resource_pool_manager, - ray_worker_group_cls=ray_worker_group_cls, - train_dataset=train_dataset, - val_dataset=val_dataset, - collate_fn=collate_fn, - train_sampler=train_sampler, - device_name=device_name) - - self.use_gpu = False - self.rl_client = create_arctic_rl_client() - self.rl_client.initialize.remote(model_name="Qwen/Qwen3-0.6B") - self.wg_kwargs["arctic_rl_client"] = self.rl_client - - - def init_workers(self): - super().init_workers() - return - # print(f"ArcticPPOTrainer.init_workers: {self.actor_rollout_wg=}") - # print(f"ArcticPPOTrainer.init_workers: {self.ref_policy_wg=}") - # print(f"ArcticPPOTrainer.init_workers: {self.async_rollout_manager=}") - # print(f"ArcticPPOTrainer.init_workers: {self.reward_loop_manager=}") - # print(f"ArcticPPOTrainer.init_workers: {self.checkpoint_manager=}") - - # self.resource_pool_manager.create_resource_pool(use_gpu=self.use_gpu) - - # self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} - - # # create actor and rollout - # actor_role = Role.ActorRolloutRef if Role.ActorRolloutRef in self.role_worker_mapping else Role.ActorRollout - # actor_rollout_resource_pool = self.resource_pool_manager.get_resource_pool(actor_role) - # actor_rollout_cls = RayClassWithInitArgs( - # cls=self.role_worker_mapping[actor_role], - # config=self.config.actor_rollout_ref, - # role=str(actor_role), - # ) - # self.resource_pool_to_cls[actor_rollout_resource_pool][str(actor_role)] = actor_rollout_cls - - # # create reference policy if needed - # # if self.use_reference_policy and Role.RefPolicy in self.role_worker_mapping: - # # resource_pool = self.resource_pool_manager.get_resource_pool(Role.RefPolicy) - # # ref_policy_cls = RayClassWithInitArgs( - # # self.role_worker_mapping[Role.RefPolicy], - # # config=self.config.actor_rollout_ref, - # # role=str(Role.RefPolicy), - # # ) - # # self.resource_pool_to_cls[resource_pool][str(Role.RefPolicy)] = ref_policy_cls - - # # initialize WorkerGroup - # # NOTE: if you want to use a different resource pool for each role, which can support different parallel size, - # # you should not use `create_colocated_worker_cls`. - # # Instead, directly pass different resource pool to different worker groups. - # # See https://github.com/volcengine/verl/blob/master/examples/ray/tutorial.ipynb for more information. - # all_wg = {} - # wg_kwargs = {} # Setting up kwargs for RayWorkerGroup - # if OmegaConf.select(self.config.trainer, "ray_wait_register_center_timeout") is not None: - # wg_kwargs["ray_wait_register_center_timeout"] = self.config.trainer.ray_wait_register_center_timeout - # if OmegaConf.select(self.config.global_profiler, "steps") is not None: - # wg_kwargs["profile_steps"] = OmegaConf.select(self.config.global_profiler, "steps") - # # Only require nsight worker options when tool is nsys - # if OmegaConf.select(self.config.global_profiler, "tool") == "nsys": - # assert ( - # OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") - # is not None - # ), "worker_nsight_options must be set when using nsys with profile_steps" - # wg_kwargs["worker_nsight_options"] = OmegaConf.to_container( - # OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") - # ) - # wg_kwargs["device_name"] = self.device_name - - # for resource_pool, class_dict in self.resource_pool_to_cls.items(): - # if not class_dict: - # continue - # worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) - # wg_dict = self.ray_worker_group_cls( - # resource_pool=resource_pool, - # ray_cls_with_init=worker_dict_cls, - # use_gpu=self.use_gpu, - # **wg_kwargs, - # ) - # spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) - # all_wg.update(spawn_wg) - - - # self.actor_rollout_wg = all_wg[str(actor_role)] - # self.actor_rollout_wg.init_model() - - # # create reward loop manager - # from verl.experimental.reward_loop import RewardLoopManager - - # # initalize reward loop manager - # # reward model (colocate or standalone): get resource_pool - # # no reward model: resource_pool = None - # resource_pool = self.resource_pool_manager.get_resource_pool(Role.RewardModel) if self.use_rm else None - # self.reward_loop_manager = RewardLoopManager( - # config=self.config, - # rm_resource_pool=resource_pool, - # ) - - # self.async_rollout_mode = True - # from verl.experimental.agent_loop import AgentLoopManager - - # # enable_agent_reward_loop = not self.use_rm or self.config.reward.reward_model.enable_resource_pool - # # reward_loop_worker_handles = self.reward_loop_manager.reward_loop_workers if enable_agent_reward_loop else None - # # self.async_rollout_manager = AgentLoopManager.create( - # # config=self.config, - # # worker_group=self.actor_rollout_wg, - # # rollout_resource_pool=actor_rollout_resource_pool, - # # reward_loop_worker_handles=reward_loop_worker_handles, - # # ) - - # self.ref_policy_wg = self.actor_rollout_wg - # self.checkpoint_manager = self.actor_rollout_wg - # self.async_rollout_manager = self.actor_rollout_wg - - - - def destroy(self): - # self.actor_rollout_wg.destroy() - self.rl_client.destroy.remote() \ No newline at end of file diff --git a/verl/trainer/ppo/ray_trainer.py b/verl/trainer/ppo/ray_trainer.py index 478bfd07908..e178ffc143d 100644 --- a/verl/trainer/ppo/ray_trainer.py +++ b/verl/trainer/ppo/ray_trainer.py @@ -309,9 +309,6 @@ def __init__( self.checkpoint_manager = None - self.wg_kwargs = {} - self.use_gpu = True - def _create_dataloader(self, train_dataset, val_dataset, collate_fn, train_sampler: Optional[Sampler]): """ Creates the train and validation dataloaders. @@ -685,7 +682,7 @@ def init_workers(self): 1. Ray resource pools from configuration 2. Worker groups for each role (actor, critic, etc.) """ - self.resource_pool_manager.create_resource_pool(use_gpu=self.use_gpu) + self.resource_pool_manager.create_resource_pool() self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} @@ -697,7 +694,6 @@ def init_workers(self): cls=self.role_worker_mapping[actor_role], config=self.config.actor_rollout_ref, role=str(actor_role), - **self.wg_kwargs, ) self.resource_pool_to_cls[actor_rollout_resource_pool][str(actor_role)] = actor_rollout_cls else: @@ -844,7 +840,6 @@ def init_workers(self): worker_group=self.actor_rollout_wg, rollout_resource_pool=actor_rollout_resource_pool, reward_loop_worker_handles=reward_loop_worker_handles, - **self.wg_kwargs, ) checkpoint_engine_config = omega_conf_to_dataclass(self.config.actor_rollout_ref.rollout.checkpoint_engine) self.checkpoint_manager = CheckpointEngineManager( @@ -1594,8 +1589,7 @@ def fit(self): metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) # TODO: implement actual tflpo and theoretical tflpo n_gpus = self.resource_pool_manager.get_n_gpus() - # To support serverless/tinker-like training, we need to support 0 GPUs training - metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=max(n_gpus, 1))) + metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) # compute variance proxy metrics gradient_norm = metrics.get("actor/grad_norm", None) metrics.update(compute_variance_proxy_metrics(batch=batch, gradient_norm=gradient_norm)) diff --git a/verl/utils/fsdp_utils.py b/verl/utils/fsdp_utils.py index 11fc247d441..8bca54fa88c 100644 --- a/verl/utils/fsdp_utils.py +++ b/verl/utils/fsdp_utils.py @@ -227,19 +227,6 @@ def load_fsdp_optimizer(optimizer, device_id): state[key] = value.to(device_id, non_blocking=True) -@torch.no_grad() -def get_fsdp_optimizer_devices(optimizer) -> list[torch.device]: - devices = set() - for param_group in optimizer.param_groups: - for param in param_group["params"]: - state = optimizer.state[param] - devices.add(param.device) - for key, value in state.items(): - if isinstance(value, torch.Tensor): - devices.add(value.device) - - return list(devices) - @contextmanager def meta_device_init(): """ diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py deleted file mode 100644 index 0882870c77e..00000000000 --- a/verl/workers/arctic_workers.py +++ /dev/null @@ -1,918 +0,0 @@ -from pathlib import Path -import torch -from verl.utils.ray_utils import auto_await -from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register -from verl.protocol import DataProto -from verl.single_controller.base import Worker -from verl.utils.profiler import DistProfiler, DistProfilerExtension -from verl.workers.engine_workers import ActorRolloutRefWorker as EngineActorRolloutRefWorker -from omegaconf import DictConfig -from tensordict import TensorDict -from dss_client.client import DSSInferenceClient, DSSTrainingClient -from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer -from deepspeed.utils import OnDevice -from verl.utils import tensordict_utils as tu -import os -import ray -from verl.utils.config import omega_conf_to_dataclass -from verl.utils.device import ( - get_device_id, - get_device_name, - get_nccl_backend, - get_torch_device, - set_expandable_segments, -) -from codetiming import Timer -import functools -import logging -import os -from contextlib import nullcontext -from functools import partial -from itertools import chain - -import torch -from codetiming import Timer -from omegaconf import DictConfig, open_dict -from tensordict import NonTensorData, TensorDict -from torch.distributed.device_mesh import init_device_mesh - -try: - from verl.workers.engine.mindspeed.transformer_impl import repatch -except ImportError: - repatch = None -from verl.checkpoint_engine import CheckpointEngineRegistry -from verl.single_controller.base import Worker -from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register -from verl.utils import tensordict_utils as tu -from verl.utils.config import omega_conf_to_dataclass -from verl.utils.device import get_device_name, set_expandable_segments -from verl.utils.distributed import initialize_global_process_group_ray -from verl.utils.flops_counter import FlopsCounter -from verl.utils.memory_utils import aggressive_empty_cache -from verl.utils.metric.utils import Metric -from verl.utils.profiler import DistProfiler, DistProfilerExtension, ProfilerConfig, log_gpu_memory_usage -from verl.utils.py_functional import append_to_dict -from verl.utils.tensordict_utils import maybe_fix_3d_position_ids -from verl.utils.torch_functional import allgather_dict_into_dict -from verl.workers.config import ActorConfig, HFModelConfig, RolloutConfig, TrainingWorkerConfig -from verl.workers.rollout.base import BaseRollout, get_rollout_class -from verl.workers.utils.losses import ppo_loss -from torch import Tensor -from verl.workers.engine.utils import postprocess_batch_func - - - -def create_meta_model(name_or_path: str): - model_config = AutoConfig.from_pretrained(name_or_path) - with OnDevice(dtype=torch.float16, device='meta'): - meta_model = AutoModelForCausalLM.from_config(model_config) - return meta_model - - -DATA_PROTO_KEYS = ["gen_batch_output", "old_log_prob", "ref_log_prob", "compute_advantage", "actor_output"] -TENSOR_DICT_KEYS = ["full_log_prob", "full_ref_log_prob", "full_actor_output"] - -def load_dump_data(train_batch_size, roll_n) -> dict[str, DataProto]: - global_step = 1 - dump_data = {} - dump_path = os.path.join('/code/users/truwase/data/at_verl_dump', f'tbs{train_batch_size}_n{roll_n}') - dump_dir = Path(dump_path) - os.path.exists(dump_dir) - for key in DATA_PROTO_KEYS: - dump_data[key] = DataProto.load_from_disk(Path(dump_dir, f"{global_step}_{key}.pt")) - for key in TENSOR_DICT_KEYS: - dump_data[key] = torch.load(Path(dump_dir, f"{global_step}_{key}.pt"), weights_only=False) - - return dump_data - - -def prepare_model_inputs_remove_padding(micro_batch: TensorDict): - from verl.utils import tensordict_utils as tu - from verl.utils.dataset.dataset_utils import DatasetPadMode - from verl.utils.debug import log_gpu_memory_usage - from verl.utils.device import get_device_id, get_device_name - from verl.utils.model import extract_multi_modal_inputs - from verl.utils.torch_functional import logprobs_from_logits - import verl.utils.torch_functional as verl_F - - use_remove_padding = tu.get_non_tensor_data(data=micro_batch, key="use_remove_padding", default=True) - pad_mode = tu.get_non_tensor_data(data=micro_batch, key="pad_mode", default=DatasetPadMode.NO_PADDING) - use_fused_kernels = tu.get_non_tensor_data(data=micro_batch, key="use_fused_kernels", default=False) - temperature = micro_batch["temperature"] - temperature_item = temperature - if use_fused_kernels: - assert not isinstance(temperature, torch.Tensor), ( - "use_fused_kernels does not support per sample temperature yet" - ) - assert pad_mode == DatasetPadMode.NO_PADDING, f"pad_mode {pad_mode} not supported" - - multi_modal_inputs = extract_multi_modal_inputs(micro_batch.get("multi_modal_inputs", [])) - input_ids = micro_batch["input_ids"] - position_ids = micro_batch["position_ids"] - - if not isinstance(temperature, torch.Tensor): - temperature = torch.tensor([temperature] * input_ids.shape[0], device=input_ids.device) - - temperature = temperature.to(torch.float32) - assert temperature.shape[0] == input_ids.shape[0] - - # args used to get outputs - output_args = {} - - # support per sample temperature - # temperature (bsz,) - # input_ids (bsz, j1) - temperature_rmpad = verl_F.expand_as_nested(temperature, input_ids).values() # (total_nnz,) - temperature_rmpad = temperature_rmpad.unsqueeze(0) # (1, total_nnz) - - if pad_mode == DatasetPadMode.NO_PADDING: - input_ids_rmpad = input_ids.values().unsqueeze(0) # (1, total_nnz) - if position_ids.dim() == 3: - position_ids_rmpad = position_ids.values().unsqueeze(1) # (4, 1, total_nnz) - else: - position_ids_rmpad = position_ids.values().unsqueeze(0) # (1, total_nnz) - else: - raise NotImplementedError(f"pad_mode {pad_mode} not implemented") - - # for compute the log_prob - input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=1) # (1, total_nnz) - - # pad and slice the inputs if sp > 1 - - input_ids_rmpad_rolled = input_ids_rmpad_rolled.squeeze(0) # ((total_nnz / sp) + pad) - temperature_rmpad = temperature_rmpad.squeeze(0) - output_args["input_ids_rmpad_rolled"] = input_ids_rmpad_rolled - output_args["temperature_rmpad"] = temperature_rmpad - - # only pass input_ids and position_ids to enable flash_attn_varlen - - model_inputs = { - "input_ids": input_ids_rmpad, - "attention_mask": None, - "position_ids": position_ids_rmpad, - "labels": input_ids_rmpad, - } - - extra_args = {} - if use_fused_kernels: - extra_args["temperature"] = temperature_item - extra_args["return_dict"] = True - - model_inputs.update(multi_modal_inputs) - model_inputs.update(extra_args) - - return model_inputs, output_args - - - -def prepare_extra_inputs(data: TensorDict) -> dict: - pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - batch_size = data["input_ids"].shape[0] - seq_len_effective = data["input_ids"].offsets().diff() - max_seq_len = max(seq_len_effective) - ready_input_ids = torch.nested.to_padded_tensor( - data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) - ) - ready_position_ids = torch.nested.to_padded_tensor( - data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) - ) - - extra_inputs = dict( - prompts=data["prompts"], - responses=data["responses"], - attention_mask=data["attention_mask"], - max_response_len=data["max_response_len"], - global_batch_size=data["global_batch_size"], - response_mask=data["response_mask"], - old_log_probs=data["old_log_probs"], - advantages=data["advantages"], - ref_log_prob=data["ref_log_prob"], - rollout_is_weights=data.get("rollout_is_weights", None), - batch_num_tokens=data["loss_mask"].sum(), - ready_input_ids=ready_input_ids, - ready_position_ids=ready_position_ids, - ready_labels=ready_input_ids, - cu_seqlens=data["input_ids"].offsets(), - ) - - - return extra_inputs - -def prepare_log_prob_extra_inputs(data: TensorDict) -> dict: - pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - batch_size = data["input_ids"].shape[0] - seq_len_effective = data["input_ids"].offsets().diff() - max_seq_len = max(seq_len_effective) - - ready_input_ids = torch.nested.to_padded_tensor( - data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) - ) - ready_position_ids = torch.nested.to_padded_tensor( - data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) - ) - - extra_inputs = dict( - ready_input_ids=ready_input_ids, - ready_position_ids=ready_position_ids, - ready_labels=ready_input_ids, - cu_seqlens=data["input_ids"].offsets() - ) - - return extra_inputs - - -def rm_padding(data: TensorDict, tensor: Tensor) -> Tensor: - cu_seqlens = data["input_ids"].offsets() - seq_lengths = cu_seqlens.diff() # (bsz,) - starts = torch.zeros_like(seq_lengths, dtype=torch.int64) # (bsz,) - tensor = torch.nested.narrow(tensor, 1, starts, seq_lengths, layout=torch.jagged) - tensor = torch.cat([t for t in tensor.unbind()]) - tensor = torch.nested.nested_tensor_from_jagged(tensor, cu_seqlens) - return tensor - -def postprocess_log_prob_output(data: TensorDict, entropy: Tensor, log_probs: Tensor) -> TensorDict: - x_entropy = rm_padding(data, entropy) - x_log_probs = rm_padding(data, log_probs) - - print(f"postprocess_log_prob_output: {x_entropy.shape=} {x_log_probs.shape=} {entropy.shape=} {log_probs.shape=}") - - micro_entropy = [t.unsqueeze(0) for t in x_entropy.unbind()] - micro_log_probs = [t.unsqueeze(0) for t in x_log_probs.unbind()] - output_lst = [] - for i in range(len(micro_entropy)): - model_output = { - "entropy": micro_entropy[i], - "log_probs": micro_log_probs[i], - } - output_lst.append({ - "model_output": model_output, - "metrics": {}, - "loss": 0.0, - }) - - return postprocess_batch_func(output_lst=output_lst, indices=None, data=data) - - -class TrainingWorker(Worker, DistProfilerExtension): - """ - TrainingWorker provides a Tinker-like API (https://thinkingmachines.ai/tinker/) as a RayWorkerGroup - to a single controller. Currently, we only provide more coarse grained APIs, - and do not provide exact APIs as Tinker does. But this can be added in the future. - """ - - def __init__(self, config: TrainingWorkerConfig, actor_config: ActorConfig, arctic_rl_client): - Worker.__init__(self) - - from verl.workers.engine import BaseEngine, EngineRegistry - - #initialize_global_process_group_ray(timeout_second=None) - - self.config = config - self.actor_config = actor_config - - self.arctic_rl_client = arctic_rl_client - - self.model_config = self.config.model_config - self.engine_config = self.config.engine_config - self.optimizer_config = self.config.optimizer_config - self.checkpoint_config = self.config.checkpoint_config - self.device_name = get_device_name() - - if self.engine_config is None: - assert self.optimizer_config is None - if self.config.auto_select_engine_optim_fn is None: - raise ValueError( - "engine_config is not provided and auto_select_engine_optim_fn is not set. " - "Cannot determine engine backend." - ) - # Support automatically select engine backend given model config - self.engine_config, self.optimizer_config = self.config.auto_select_engine_optim_fn( - self.model_config, self.device_name - ) - - # we use the one defined in model - # TODO: this is not elegant and should refactor later - self.engine_config.use_remove_padding = self.model_config.use_remove_padding - self.engine_config.use_fused_kernels = self.model_config.use_fused_kernels - - if repatch is not None: - # NPU MindSpeed patch, will be refactored with MindSpeedEngine. - repatch(self.engine_config.get("override_transformer_config", {})) - - # TODO: add DistProfilerExtension - self.profiler_config = self.config.profiler_config - if self.profiler_config is not None: - self.profiler_tool_config = self.profiler_config.tool_config.get(self.profiler_config.tool, {}) - else: - self.profiler_tool_config = None - - DistProfilerExtension.__init__( - self, DistProfiler(rank=self.rank, config=self.profiler_config, tool_config=self.profiler_tool_config) - ) - - # self.engine: BaseEngine = EngineRegistry.new( - # model_type=self.config.model_type, - # backend=self.engine_config.strategy, - # model_config=self.model_config, - # engine_config=self.engine_config, - # optimizer_config=self.optimizer_config, - # checkpoint_config=self.checkpoint_config, - # ) - - # # build dispatch info - # self._register_dispatch_collect_info( - # mesh_name="train", - # dp_rank=self.engine.get_data_parallel_rank(), - # is_collect=self.engine.is_mp_src_rank_with_outputs(), - # ) - - self.flops_counter = FlopsCounter(self.model_config.hf_config) - - self.loss_fn = None - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def to(self, device, model=True, optimizer=True, grad=True): - """Manual control of load/offload""" - assert device in ["cpu", "device"] - - if device == "device": - device = get_device_name() - - self.engine.to(device=device, model=model, optimizer=optimizer, grad=grad) - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def set_loss_fn(self, loss_fn): - self.loss_fn = loss_fn - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def reset(self): - """ - Reset the model engine to the initial state. If the engine is not initialized, - we initialize it. Otherwise, reload ckpt and reset states - """ - pass # self.engine.initialize() - - def _postprocess_output(self, output, *, global_token_num, delta_time, forward_only, images_seqlens): - """ - - Args: - output: a dictionary containing loss, model_outputs and metrics - - Returns: - - """ - # TODO: whether to log memory - # metrics["perf/max_memory_allocated_gb"] = get_torch_device().max_memory_allocated() / (1024 ** 3) - # metrics["perf/max_memory_reserved_gb"] = get_torch_device().max_memory_reserved() / (1024 ** 3) - # metrics["perf/cpu_memory_used_gb"] = psutil.virtual_memory().used / (1024 ** 3) - - metrics: dict = output.pop("metrics") - # perform all gather in dp group to ensure that it's correct. - # Here each metric in metrics can be a list (micro-batch metrics) or a singleton - # we should always sum the loss of each micro-batch as we scale by global_bsz/global_token - loss = torch.sum(torch.tensor(output.pop("loss"), device=self.device_name)) - - # For grad_norm, we do not perform all reduce because it is already been done when clipping grad - grad_norm = metrics.pop("grad_norm", None) - lr = metrics.pop("lr", None) - - final_metrics = metrics - - final_metrics["loss"] = loss - if grad_norm is not None: - final_metrics["grad_norm"] = grad_norm - if lr is not None: - final_metrics["lr"] = lr - - # TODO: confirm the mtp loss IS same across dp - for k, v in final_metrics.items(): - if k.startswith("mtp_losses"): - flatten_v = [sublist[0] for sublist in v] # sublist should be single element - final_metrics[k] = sum(flatten_v) / len(flatten_v) - # compute mfu - if global_token_num is not None: - estimated_flops, promised_flops = self.flops_counter.estimate_flops( - global_token_num, delta_time, images_seqlens=images_seqlens - ) - final_metrics["mfu"] = estimated_flops / promised_flops - if forward_only: - final_metrics["mfu"] /= 3.0 - # model outputs - model_output = output.pop("model_output", {}) - # We only return final_metrics - final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": final_metrics}) - return final_output - - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train"), blocking=False) - def train_mini_batch(self, data: TensorDict) -> TensorDict: - """Split a batch into N mini-batches run for multiple epochs - - Args: - data: - - Returns: - - """ - batch_size_per_dp = data.shape[0] - disable_auto_offload = tu.pop(data, key="disable_auto_offload", default=False) - mini_batch_size = tu.pop(data, key="mini_batch_size", default=None) - num_mini_batch = tu.pop(data, key="num_mini_batch", default=None) - epochs = tu.pop(data, key="epochs", default=1) - seed = tu.pop(data, key="seed", default=42) - dataloader_kwargs = tu.pop(data, key="dataloader_kwargs", default={}) - - self.engine_config = self.config.engine_config - - assert mini_batch_size is not None or num_mini_batch is not None - - mini_batch_size_per_gpu = mini_batch_size - - # make iterator - dataloader = tu.make_iterator( - data, - mini_batch_size=mini_batch_size_per_gpu, - epochs=epochs, - seed=seed, - dataloader_kwargs=dataloader_kwargs, - ) - - with ( - Timer(name="train_batch", logger=None), - ): - # update - output_lst = [] - total_num_iterations = data.shape[0] // mini_batch_size_per_gpu * epochs - - for batch_idx, mini_batch_td in enumerate(dataloader): - # add global token num - global_token_num = mini_batch_td["input_ids"].offsets().diff().tolist() # (total_nnz,) - tu.assign_non_tensor( - mini_batch_td, - global_token_num=NonTensorData(global_token_num), - update_lr_scheduler=batch_idx == total_num_iterations - 1, - disable_auto_offload=True, - ) - actor_output = self.train_batch(mini_batch_td) - output_lst.append(actor_output) - - actor_output = [tu.get(output, "metrics") for output in output_lst] - metrics = {} - for output in actor_output: - for key, val in output.items(): - print(f"metrics {key=} {val=}") - - # flattn dp and micro batch - if isinstance(val, list): - output[key] = ( - Metric.aggregate_dp(val) - if isinstance(val[0], Metric) - else list(chain.from_iterable(val)) - ) - append_to_dict(metrics, output) - - output = tu.get_tensordict(tensor_dict={}, non_tensor_dict={"metrics": metrics}).cpu() - - return output - - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train"), blocking=False) - def train_batch(self, data: TensorDict) -> TensorDict: - assert self.loss_fn is not None, "loss function can't be None when calling train_batch" - - # global_token_num should be a list of number of tokens of each seq in this batch - global_token_num = tu.get(data, key="global_token_num") - disable_auto_offload = tu.get(data, key="disable_auto_offload", default=False) - images_seqlens = tu.get(data, key="images_seqlens", default=None) - - # inject engineering parameters if not specified - default_keys = dict( - use_remove_padding=self.model_config.use_remove_padding, - use_dynamic_bsz=self.engine_config.use_dynamic_bsz, - max_token_len_per_gpu=self.engine_config.max_token_len_per_gpu, - micro_batch_size_per_gpu=self.engine_config.micro_batch_size_per_gpu, - use_fused_kernels=self.engine_config.use_fused_kernels, - ) - - for key, val in default_keys.items(): - if key not in data.keys(): - tu.assign_non_tensor(data, **{key: val}) - - with ( - Timer(name="train_batch", logger=None) as timer, - ): - # XXX: what's missing is the loss function to be run on the dss side - # arctic-verl/verl/workers/engine/fsdp/transformer_impl.py:1098 forward_step - # the loss function is arctic-verl/verl/workers/utils/losses.py:97 ppo_loss - # from verl.workers.utils.losses import ppo_loss <- need to adapt to pass a gazillion of config variables - - # from verl.utils.tensordict_utils import chunk_tensordict - # batch = chunk_tensordict(data, 1) - print(f"update_actor data: {data}") - - # XXX: fixme - # batch = batch[0] - - dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) - print(f"{dss_batch_dict=}") - print(f"{output_args=}") - # import pdb; pdb.set_trace() - - # we need to serialize the config object to dict - # dataclasses.asdict only returns keys that are defined at init (vars will do more) - but perhaps we want `asdict`? - actor_config_as_dict = vars(self.actor_config) - print(f"update_actor: {self.actor_config=}") - print(f"update_actor: {actor_config_as_dict}") - import json - def safe_serialize(obj): - return json.loads(json.dumps(obj, default=lambda o: None)) - #actor_config_as_dict = safe_serialize(self.actor_config) - actor_config_as_dict = safe_serialize(actor_config_as_dict) - - - # pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - # batch_size = data["input_ids"].shape[0] - # seq_len_effective = data["input_ids"].offsets().diff() - # max_seq_len = max(seq_len_effective) - # ready_input_ids = torch.nested.to_padded_tensor( - # data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) - # ) - # ready_position_ids = torch.nested.to_padded_tensor( - # data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) - # ) - # extra_inputs = dict( - # prompts=data["prompts"], - # responses=data["responses"], - # attention_mask=data["attention_mask"], - # max_response_len=data["max_response_len"], - # global_batch_size=data["global_batch_size"], - # response_mask=data["response_mask"], - # old_log_probs=data["old_log_probs"], - # advantages=data["advantages"], - # ref_log_prob=data["ref_log_prob"], - # rollout_is_weights=data.get("rollout_is_weights", None), - # ready_input_ids=ready_input_ids, - # ready_position_ids=ready_position_ids, - # ready_labels=ready_input_ids, - # # =batch[""], - # ) - # extra_inputs["batch_num_tokens"] = data["loss_mask"].sum() - - extra_inputs = prepare_extra_inputs(data) - policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) - - post_process_inputs = dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config, extra_inputs=extra_inputs) - print(f"update_actor: {post_process_inputs=}") - - # XXX: pass the original batch as post_process_inputs["batch"] - the ppo loss function expects data["prompts"] - # it got stripped and is not in dss_batch_dict -# File "/code/users/stas/github/sf/dss-platform/dss/processors/verl.py", line 90, in fwd_post_process_ppo_loss -# return ppo_loss(config, model_output, data) -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -# File "/code/users/stas/github/sf/dss-platform/dss/processors/verl.py", line 100, in ppo_loss -# log_prob = no_padding_2_padding(model_output["log_probs"], data) -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -# File "/code/users/stas/github/sf/arctic-verl/verl/workers/utils/padding.py", line 99, in no_padding_2_padding -# prompt_ids = data["prompts"] -# ~~~~^^^^^^^^^^^ -# KeyError: 'prompts' - - loss, metrics = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) - # output = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) - print(f"update_actor: {loss=}") - print(f"update_actor: {metrics=}") - - - from verl.utils.metric import AggregationType, Metric - # XXX: fix me - we need to aggregate the metrics - metrics = {k:Metric(value=v[0], aggregation=AggregationType.MEAN) for k,v in metrics.items()} - - delta_time = timer.last - - # XXX: fix me - # metrics = { - # 'actor/pg_clipfrac': None, - # 'actor/ppo_kl': None, - # 'actor/pg_clipfrac_lower': None, - # 'actor/pg_loss': None, - # 'kl_loss': None, - # 'kl_coef': None, - # 'grad_norm': None, - # } - - print(f"{data=}") - print(f"{data["input_ids"].shape=}") - model_output = { - # XXX: fix me - made a copy of existing same shape tensor for now - # 'log_probs': batch[0]["ref_log_prob"] - } - - # expected output so far - # - # output={ - # 'model_output': { - # 'log_probs': NestedTensor(size=(1,j18), offsets=tensor([ 0,401], device='cuda:0'), grad_fn=, contiguous=True) - # }, - # 'loss': [-0.9999991059303284], - # 'metrics': { - # 'actor/pg_clipfrac': , - # 'actor/ppo_kl': , - # 'actor/pg_clipfrac_lower': , - # 'actor/pg_loss': , - # 'kl_loss': , - # 'kl_coef': [0.001], - # 'grad_norm': 16.321151733398438, - # } - # } - - - #output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict=non_tensor_dict) - output = dict( - model_output=model_output, - metrics=metrics, - loss=loss, - ) - - update_lr_scheduler = tu.get(data, key="update_lr_scheduler", default=False) - # XXX: fix me - update_lr_scheduler = False - # update lr scheduler - if update_lr_scheduler: - lr = self.engine.lr_scheduler_step() - else: - lr = None - - - # we don't need model_output in training. Maybe we change out mind later - #output.pop("model_output") - if lr is not None: - output["metrics"]["lr"] = lr - - final_output = self._postprocess_output( - output, - global_token_num=global_token_num, - delta_time=delta_time, - forward_only=False, - images_seqlens=images_seqlens, - ).cpu() - - return final_output - - - - - - -class ActorRolloutRefWorker(Worker, DistProfilerExtension): - def __init__(self, config: DictConfig, role: str, **kwargs): - Worker.__init__(self) - self.config = config - self.role = role - self._is_actor = self.role in ["actor", "actor_rollout", "actor_rollout_ref"] - self._is_rollout = self.role in ["rollout", "actor_rollout", "actor_rollout_ref"] - self._is_ref = self.role in ["ref", "actor_rollout_ref"] - - self.arctic_rl_client = kwargs.get("arctic_rl_client", None) - - # assert self.arctic_rl_client is not None, "arctic_rl_client is required" - self._loaded_dump_data = load_dump_data(1, 1) - DistProfilerExtension.__init__(self, DistProfiler(rank=self.rank, config=None, tool_config=None)) - - if self._is_actor: - model_config: HFModelConfig = omega_conf_to_dataclass(self.config.model) - actor_config: ActorConfig = omega_conf_to_dataclass(self.config.actor) - actor_config.model_config = model_config - actor_training_config = TrainingWorkerConfig( - model_type="language_model", - model_config=actor_config.model_config, - engine_config=actor_config.engine, - optimizer_config=actor_config.optim, - checkpoint_config=actor_config.checkpoint, - ) - self.actor_config = actor_config - - assert self.config.actor.use_dynamic_bsz == self.config.rollout.log_prob_use_dynamic_bsz - - # assign engine configs - actor_training_config.engine_config.use_dynamic_bsz = self.config.actor.use_dynamic_bsz - actor_training_config.engine_config.infer_max_token_len_per_gpu = ( - self.config.rollout.log_prob_max_token_len_per_gpu - ) - actor_training_config.engine_config.infer_micro_batch_size_per_gpu = ( - self.config.rollout.log_prob_micro_batch_size_per_gpu - ) - actor_training_config.engine_config.max_token_len_per_gpu = self.config.actor.ppo_max_token_len_per_gpu - actor_training_config.engine_config.micro_batch_size_per_gpu = ( - self.config.actor.ppo_micro_batch_size_per_gpu - ) - actor_training_config.engine_config.use_remove_padding = model_config.use_remove_padding - - if self.config.actor.use_dynamic_bsz: - assert self.config.rollout.log_prob_max_token_len_per_gpu is not None - assert self.config.actor.ppo_max_token_len_per_gpu is not None - else: - assert self.config.rollout.log_prob_micro_batch_size_per_gpu is not None - assert self.config.actor.ppo_micro_batch_size_per_gpu is not None - - self.actor = TrainingWorker(config=actor_training_config, actor_config=actor_config, arctic_rl_client=self.arctic_rl_client) - - self.actor.reset() - self.loss_fn = partial(ppo_loss, config=actor_config) - self.actor.set_loss_fn(loss_fn=self.loss_fn) - - self.set_dispatch_collect(mesh_name="actor", **self.actor.get_dispatch_collect()) - - # from verl.workers.actor import DataParallelPPOActor - - # # hacks to appease to DataParallelPPOActor - # import torch.distributed - # torch.distributed.get_rank = lambda: 0 - - # actor_cfg = omega_conf_to_dataclass(self.config.actor) - # self.actor = DataParallelPPOActor( - # # XXX: hijack actor_module - # config=actor_cfg, actor_module=None, actor_optimizer=None - # ) - - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - self._register_dispatch_collect_info("actor", dp_rank=self.rank, is_collect=True) - self._register_dispatch_collect_info("ref", dp_rank=self.rank, is_collect=True) - self._register_dispatch_collect_info("rollout", dp_rank=self.rank, is_collect=True) - - return - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def destroy(self): - self.dss_training_engine.destroy() - self.arctic_inference_engine.destroy() - return - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def set_loss_fn(self, loss_fn): - return - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def to(self, device, model=True, optimizer=True, grad=True): - """Manual control of load/offload""" - return - - - def _update_config_params(self, data: TensorDict): - default_keys = dict( - use_remove_padding=self.actor.model_config.use_remove_padding, - use_dynamic_bsz=self.actor.engine_config.use_dynamic_bsz, - max_token_len_per_gpu=self.actor.engine_config.max_token_len_per_gpu, - micro_batch_size_per_gpu=self.actor.engine_config.micro_batch_size_per_gpu, - use_fused_kernels=self.actor.engine_config.use_fused_kernels, - ) - - for key, val in default_keys.items(): - if key not in data.keys(): - tu.assign_non_tensor(data, **{key: val}) - - - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="ref")) - @DistProfiler.annotate(color="olive", role="ref_compute_log_prob") - def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: - # return self._loaded_dump_data["full_ref_log_prob"] - # import pdb; pdb.set_trace() - # from verl.utils.tensordict_utils import chunk_tensordict - # batch = chunk_tensordict(data, 1) - # print(f"compute_ref_log_prob data: {data}") - dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) - # print(f"{dss_batch_dict=}") - # import pdb; pdb.set_trace() - # self.dss_training_engine.forward(**dss_batch_dict) - # loss = self.dss_training_engine.backward() - # print(f"loss: {loss}") - # import pdb; pdb.set_trace() - # log_prob = self._loaded_dump_data["full_log_prob"] - - self._update_config_params(data) - post_process_inputs = prepare_log_prob_extra_inputs(data) - entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) - - batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) - model_output = batch_output.pop("model_output", {}) - metrics = { - "mfu": 0.0, - "loss": 1.0, - } - final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) - - - # metrics = { - # "mfu": 0.0, - # "loss": 1.0, - # "batch_size": 1, - # } - - # model_output = { - # "log_probs": log_probs, - # } - - # final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) - - return final_output - - - - - # TODO: Actor API Begin - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) - @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") - def compute_log_prob(self, data: TensorDict) -> TensorDict: - # import pdb; pdb.set_trace() - # from verl.utils.tensordict_utils import chunk_tensordict - # batch = chunk_tensordict(data, 1) - # print(f"compute_log_prob data: {data}") - dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) - # print(f"{dss_batch_dict=}") - # import pdb; pdb.set_trace() - # self.dss_training_engine.forward(**dss_batch_dict) - # loss = self.dss_training_engine.backward() - # print(f"loss: {loss}") - # import pdb; pdb.set_trace() - # log_prob = self._loaded_dump_data["full_log_prob"] - - self._update_config_params(data) - post_process_inputs = prepare_log_prob_extra_inputs(data) - # print(f"compute_log_prob: {post_process_inputs=}") - - entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) - - # import pdb; pdb.set_trace() - batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) - model_output = batch_output.pop("model_output", {}) - metrics = { - "mfu": 0.0, - "loss": 1.0, - } - final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) - - return final_output - - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) - @DistProfiler.annotate(color="red", role="actor_update") - def update_actor(self, data: TensorDict) -> TensorDict: - output = self.actor.train_mini_batch(data=data) - return output.cpu() if output is not None else None - - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def load_checkpoint(self, local_path, hdfs_path=None, del_local_after_load=False): - assert "actor" in self.role, "load_checkpoint only support actor role" - return - - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): - assert "actor" in self.role, "save_checkpoint only support actor role" - return - - @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) - async def update_weights(self, global_steps: int = None): - """Update weights from trainer to rollout. - - 1. For sync training with colocated trainer and rollout, update rollout directly from model engine. - - before update_weights: rollout should be in sleep mode. - - after update_weights: rollout should be in wake_up mode. - 2. For async training with disaggregated trainer and rollout, send_weights only by checkpoint engine. - """ - return - - # TODO: Actor API End - - - # TODO: Rollout API Begin - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="rollout")) - async def generate_sequences(self, batch: DataProto): - print(f"{batch.non_tensor_batch=}") - raw_prompts = list(batch.non_tensor_batch["raw_prompt"]) - print(f"{raw_prompts=}") - prompts = self.tokenizer.apply_chat_template( - raw_prompts, - add_generation_prompt=True, - tokenize=False, - ) - # import pdb; pdb.set_trace() - print(f"prompts: {prompts}") - metrics = {} - - gen_batch_output = self.arctic_inference_engine.generate(prompts=prompts) - - return gen_batch_output - # return self._loaded_dump_data["gen_batch_output"] - - # TODO: Rollout API End - - # TODO: CheckpointManager API Begin - @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) - async def sleep_replicas(self): - """Sleep all rollout replicas: free weight and kv_cache device memory.""" - return - # TODO: CheckpointManager API \ No newline at end of file diff --git a/verl/workers/engine_workers.py b/verl/workers/engine_workers.py index 5367479a5bf..d0c065e4dfd 100644 --- a/verl/workers/engine_workers.py +++ b/verl/workers/engine_workers.py @@ -667,7 +667,7 @@ async def update_weights(self, global_steps: int = None): log_gpu_memory_usage("After update_weights", logger=logger) # 3. offload model to cpu - self.actor.engine.to("cpu", model=self.actor.engine.is_param_offload_enabled, optimizer=False, grad=False) + self.actor.engine.to("cpu", model=True, optimizer=False, grad=False) aggressive_empty_cache(force_sync=True) # 4. resume kv_cache diff --git a/verl/workers/rollout/arctic_rollout/__init__.py b/verl/workers/rollout/arctic_rollout/__init__.py deleted file mode 100644 index cf453f8b2f6..00000000000 --- a/verl/workers/rollout/arctic_rollout/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .arctic_rollout import ArcticReplica - -__all__ = ["ArcticReplica"] diff --git a/verl/workers/rollout/arctic_rollout/arctic_rollout.py b/verl/workers/rollout/arctic_rollout/arctic_rollout.py deleted file mode 100644 index c744b4a00fd..00000000000 --- a/verl/workers/rollout/arctic_rollout/arctic_rollout.py +++ /dev/null @@ -1,322 +0,0 @@ -import ray -from typing import Any, Optional -from verl.workers.rollout.vllm_rollout.vllm_async_server import vLLMHttpServer - -import argparse -from typing import Any, Optional -from verl.trainer.ppo.arctic_rl_client import ArcticRLClient4VeRL -from collections.abc import AsyncGenerator - -import ray -from ray.actor import ActorHandle -from vllm import SamplingParams -from vllm.inputs import TokensPrompt -from vllm.lora.request import LoRARequest -from vllm.outputs import RequestOutput, CompletionOutput - -from verl.utils.tokenizer import normalize_token_ids -from verl.workers.config import HFModelConfig, RolloutConfig -from verl.workers.rollout.replica import RolloutMode, RolloutReplica, TokenOutput -from verl.workers.rollout.vllm_rollout.utils import ( - VLLM_LORA_INT_ID, - VLLM_LORA_NAME, - VLLM_LORA_PATH, -) -from transformers import AutoTokenizer -from verl.utils.config import omega_conf_to_dataclass -from verl.workers.rollout.utils import get_max_position_embeddings - - -class ArcticLLMEngine: - def __init__( - self, - replica_rank: int, - arctic_rl_client: ArcticRLClient4VeRL, - ): - self.replica_rank = replica_rank - self.arctic_rl_client = arctic_rl_client - self.tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") - - async def generate( - self, - prompt: TokensPrompt, - sampling_params: dict[str, Any], - request_id: str, - lora_request: Optional[LoRARequest] = None, - priority: int = 0, - ) -> AsyncGenerator[RequestOutput, None]: - gen_batch_output = await self.arctic_rl_client.generate.remote( - prompt_ids=prompt['prompt_token_ids'], - sampling_params=sampling_params, - ) - # print(f"arctic_async_server: {gen_batch_output=}, {type(gen_batch_output)=}") - # gen_batch_output = await ray.get(gen_batch_output) - - raw_prompt = self.tokenizer.decode(prompt['prompt_token_ids']) - completed_outputs = [] - for i, output in enumerate(gen_batch_output): - completed_outputs.append(CompletionOutput( - index=i, - text=output['text'], - token_ids=output['token_ids'], - finish_reason=output['finish_reason'], - cumulative_logprob=None, - logprobs=None - ) - ) - - yield RequestOutput( - request_id=request_id, - outputs=completed_outputs, - prompt=raw_prompt, - prompt_logprobs=None, - prompt_token_ids=prompt['prompt_token_ids'], - # finished=completed_output.finish_reason == "stop", - finished=True, - ) - - -class ArcticLLMServer(vLLMHttpServer): - """vLLM http server in single node, this is equivalent to launch server with command line: - ``` - vllm serve --tensor-parallel-size=8 ... - ``` - """ - - def __init__( - self, - config: RolloutConfig, - model_config: HFModelConfig, - rollout_mode: RolloutMode, - arctic_rl_client: ArcticRLClient4VeRL, - workers: list[ActorHandle] = [], - replica_rank: int = 0, - node_rank: int = 0, - gpus_per_node: int = 1, - nnodes: int = 1, - cuda_visible_devices: str = "0", - ): - """ - Args: - config (RolloutConfig): full config. - model_config (HFModelConfig): model config. - rollout_mode (RolloutMode): rollout mode. - replica_rank (int): replica rank, a replica may contain multiple nodes. - node_rank (int): node rank. - gpus_per_node (int): number of gpus per node. - nnodes (int): number of nodes. - cuda_visible_devices (str): cuda visible devices. - """ - self.config: RolloutConfig = omega_conf_to_dataclass(config) - self.model_config: HFModelConfig = omega_conf_to_dataclass(model_config, dataclass_type=HFModelConfig) - max_position_embeddings = get_max_position_embeddings(self.model_config.hf_config) - if self.config.max_model_len is None: - self.config.max_model_len = max_position_embeddings - else: - if self.config.max_model_len > max_position_embeddings: - raise ValueError( - f"max_model_len ({self.config.max_model_len}) should be less than or equal to " - f"max_position_embeddings ({max_position_embeddings})" - ) - - self.rollout_mode = rollout_mode - self.workers = workers - - self.replica_rank = replica_rank - self.node_rank = node_rank - self.gpus_per_node = gpus_per_node - self.nnodes = nnodes - # model weights version, set by ServerAdapter when update weights. - self.global_steps = None - - if self.rollout_mode != RolloutMode.HYBRID and self.config.load_format == "dummy": - # logger.warning(f"rollout mode is {self.rollout_mode}, load_format is dummy, set to auto") - self.config.load_format = "auto" - - - self._master_address = None - self._master_port = None - self._dp_rpc_port = None - self._dp_master_port = None - - self.engine = ArcticLLMEngine(replica_rank, arctic_rl_client) - - # logger.info( - # f"vLLMHttpServer, replica_rank: {self.replica_rank}, node_rank: {self.node_rank}, " - # f"{get_visible_devices_keyword()}: {cuda_visible_devices}, " - # f"master_address: {self._master_address}, master_port: {self._master_port}, " - # f"data_parallel_rpc_port: {self._dp_rpc_port}, data_parallel_master_port: {self._dp_master_port}" - # ) - - def get_master_address(self): pass - - def get_server_address(self): pass - - @property - def lora_as_adapter(self) -> bool: pass - - async def collective_rpc( - self, - **kwargs, - ): - pass - - async def launch_server(self, master_address: str = None, master_port: int = None, dp_rpc_port: int = None): - pass - - async def run_server(self, args: argparse.Namespace): - pass - - - async def generate( - self, - prompt_ids: list[int], - sampling_params: dict[str, Any], - request_id: str, - image_data: Optional[list[Any]] = None, - video_data: Optional[list[Any]] = None, - priority: int = 0, - ) -> TokenOutput: - """Generate sequence with token-in-token-out.""" - prompt_ids = normalize_token_ids(prompt_ids) - - # Calculate the maximum possible new tokens based on available context space - # This serves as a safety upper bound - max_possible_tokens = self.config.max_model_len - len(prompt_ids) - if max_possible_tokens < 0: - raise ValueError( - f"Prompt length ({len(prompt_ids)}) exceeds the model's maximum context length " - f"({self.config.max_model_len})." - ) - - # Determine max_tokens from sampling_params or use configured response_length as default - if "max_tokens" in sampling_params: - max_tokens = sampling_params.pop("max_tokens") - elif "max_new_tokens" in sampling_params: - # support sglang-style 'max_new_tokens' param - max_tokens = sampling_params.pop("max_new_tokens") - else: - # Default to a calculation that considers configured lengths - max_tokens = self.config.response_length + self.config.prompt_length - len(prompt_ids) - - # Clamp max_tokens to the valid range [0, max_possible_tokens] - max_tokens = max(0, min(max_tokens, max_possible_tokens)) - - assert max_tokens <= max_possible_tokens, ( - f"max_tokens {max_tokens} exceeds available context space {max_possible_tokens}" - ) - sampling_params["logprobs"] = 0 if sampling_params.pop("logprobs", False) else None - sampling_params.setdefault("repetition_penalty", self.config.get("repetition_penalty", 1.0)) - # sampling_params = SamplingParams(max_tokens=max_tokens, **sampling_params) - sampling_params["max_tokens"] = max_tokens - multi_modal_data = {} - if image_data is not None: - multi_modal_data["image"] = image_data - if video_data is not None: - multi_modal_data["video"] = video_data - # import pdb; pdb.set_trace() - prompt = TokensPrompt(prompt_token_ids=prompt_ids, multi_modal_data=multi_modal_data) - - # Add lora request - lora_request = None - if self.lora_as_adapter: - # Make sure we also check that the lora is already loaded in the engine - lora_loaded = VLLM_LORA_INT_ID in await self.engine.list_loras() - if lora_loaded: - lora_request = LoRARequest( - lora_name=VLLM_LORA_NAME, lora_int_id=VLLM_LORA_INT_ID, lora_path=VLLM_LORA_PATH - ) - # import pdb; pdb.set_trace() - generator = self.engine.generate( - prompt=prompt, - sampling_params=sampling_params, - request_id=request_id, - lora_request=lora_request, - priority=priority, - ) - - # print(f"arctic_async_server: {generator=}, {type(generator)=}") - - # Get final response - final_res: Optional[RequestOutput] = None - async for output in generator: - final_res = output - assert final_res is not None - - token_ids = final_res.outputs[0].token_ids - log_probs = None - if sampling_params["logprobs"] is not None: - log_probs = [logprobs[token_ids[i]].logprob for i, logprobs in enumerate(final_res.outputs[0].logprobs)] - - routed_experts = None - if self.config.enable_rollout_routing_replay: - routed_experts = final_res.outputs[0].routed_experts - - # Determine stop reason from finish_reason - finish_reason = final_res.outputs[0].finish_reason - if finish_reason == "abort": - stop_reason = "aborted" - elif finish_reason in ("stop", "length"): - stop_reason = "completed" - else: - stop_reason = finish_reason # for more stop reason in the future - - num_preempted = None - - if hasattr(final_res.outputs[0], "num_preempted"): - num_preempted = final_res.outputs[0].num_preempted - - return TokenOutput( - token_ids=token_ids, - log_probs=log_probs, - routed_experts=routed_experts, - stop_reason=stop_reason, - num_preempted=num_preempted, - extra_info={"global_steps": self.global_steps}, - ) - - - - -class ArcticReplica(RolloutReplica): - def __init__( - self, - replica_rank: int, - config: RolloutConfig, - model_config: HFModelConfig, - gpus_per_node: int = 1, - is_reward_model: bool = False, - **kwargs, - ): - super().__init__(replica_rank, config, model_config, gpus_per_node, is_reward_model) - self.server_class = ray.remote(ArcticLLMServer) - self.arctic_rl_client = kwargs.get("arctic_rl_client", None) - # assert self.arctic_rl_client is not None, "arctic_rl_client is required" - - - def rollout_worker_use_gpu(self) -> bool: - return False - - - async def launch_servers(self): - server = self.server_class.options( - ).remote( - replica_rank=self.replica_rank, - config=self.config, - model_config=self.model_config, - rollout_mode=self.rollout_mode, - arctic_rl_client=self.arctic_rl_client, - ) - self.servers.append(server) - self._server_handle = server - - - async def wake_up(self): - pass - - async def sleep(self): - pass - - async def abort_request(self, request_id: str) -> dict[str, Any]: - return {"aborted": True, "request_id": 0} - \ No newline at end of file diff --git a/verl/workers/rollout/replica.py b/verl/workers/rollout/replica.py index 2557eb74d7a..969c6208083 100644 --- a/verl/workers/rollout/replica.py +++ b/verl/workers/rollout/replica.py @@ -348,16 +348,11 @@ def _load_trtllm(): return TRTLLMReplica -def _load_arctic(): - from verl.workers.rollout.arctic_rollout.arctic_rollout import ArcticReplica - - return ArcticReplica # Register built-in types RolloutReplicaRegistry.register("vllm", _load_vllm) RolloutReplicaRegistry.register("sglang", _load_sglang) RolloutReplicaRegistry.register("trtllm", _load_trtllm) -RolloutReplicaRegistry.register("arctic", _load_arctic) # Original function for backward compatibility From 6dec5a2380903720b7131244183d81e7642f7f49 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 2 Apr 2026 17:44:42 +0000 Subject: [PATCH 04/58] Merge ARL WIP --- examples/arctic_rl/run_arctic_gsm8k_grpo.sh | 86 ++ examples/arctic_rl/run_gsm8k_grpo.sh | 81 ++ verl/experimental/agent_loop/agent_loop.py | 9 +- verl/single_controller/ray/base.py | 15 +- verl/trainer/config/ppo_trainer.yaml | 3 + verl/trainer/main_ppo.py | 15 +- verl/trainer/ppo/arctic_rl_client.py | 130 +++ verl/trainer/ppo/arctic_trainer.py | 153 +++ verl/trainer/ppo/ray_trainer.py | 10 +- verl/utils/fsdp_utils.py | 13 + verl/workers/arctic_workers.py | 918 ++++++++++++++++++ verl/workers/engine_workers.py | 2 +- .../rollout/arctic_rollout/__init__.py | 3 + .../rollout/arctic_rollout/arctic_rollout.py | 322 ++++++ verl/workers/rollout/replica.py | 5 + 15 files changed, 1750 insertions(+), 15 deletions(-) create mode 100755 examples/arctic_rl/run_arctic_gsm8k_grpo.sh create mode 100755 examples/arctic_rl/run_gsm8k_grpo.sh create mode 100644 verl/trainer/ppo/arctic_rl_client.py create mode 100644 verl/trainer/ppo/arctic_trainer.py create mode 100644 verl/workers/arctic_workers.py create mode 100644 verl/workers/rollout/arctic_rollout/__init__.py create mode 100644 verl/workers/rollout/arctic_rollout/arctic_rollout.py diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh new file mode 100755 index 00000000000..44cc760c708 --- /dev/null +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +# we want to make sure this runs on non-gpu client +export CUDA_VISIBLE_DEVICES= + +# BSZ=1024 +BSZ=2 +MBS=2 +UBS=2 +ROLL_N=2 +MAX_STEPS=1 +# LR=0 +LR=1e-6 +LOGGER=console +# LOGGER="['console','wandb']" +USE_KL_LOSS=True +# USE_KL_LOSS=False +# REMOVE_PADDING=True +REMOVE_PADDING=False +MODEL="Qwen/Qwen3-0.6B" +# STRATEGY="fsdp" +STRATEGY="fsdp2" +PYTHONUNBUFFERED=1 +HYDRA_FULL_ERROR=1 +USE_LEGACY_WORKER_IMPL=disable +NGPU_PER_NODE=1 +ROLLOUT_NAME=arctic # entry point into ArcticRL +USE_ARCTIC_RL=True + +experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_at${USE_ARCTIC_RL}" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=/code/shared/gsm8k/train.parquet \ + data.val_files=/code/shared/gsm8k/test.parquet \ + data.train_batch_size=${BSZ} \ + data.max_prompt_length=64 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + reward.num_workers=1 \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.model.path=${MODEL} \ + actor_rollout_ref.actor.optim.lr=${LR} \ + actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.strategy=${STRATEGY} \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.n=${ROLL_N} \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.ref.fsdp_config.param_offload=False \ + actor_rollout_ref.ref.strategy=${STRATEGY} \ + algorithm.use_kl_in_reward=False \ + trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ + trainer.use_arctic_rl=${USE_ARCTIC_RL} \ + trainer.critic_warmup=0 \ + trainer.logger=${LOGGER} \ + trainer.experiment_name=${experiment_name} \ + trainer.project_name='verl_arctic_grpo_gsm8k' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_training_steps=${MAX_STEPS} \ + trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log + + # trainer.total_training_steps=${MAX_STEPS} \ diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh new file mode 100755 index 00000000000..f65b0f55b24 --- /dev/null +++ b/examples/arctic_rl/run_gsm8k_grpo.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +# BSZ=1024 +BSZ=2 +MBS=2 +UBS=2 +ROLL_N=2 +MAX_STEPS=1 +# LR=0 +LR=1e-6 +LOGGER=console +# LOGGER="['console','wandb']" +USE_KL_LOSS=True +# USE_KL_LOSS=False +# REMOVE_PADDING=True +REMOVE_PADDING=False +MODEL="Qwen/Qwen3-0.6B" +# STRATEGY="fsdp" +STRATEGY="fsdp2" +PYTHONUNBUFFERED=1 +HYDRA_FULL_ERROR=1 +USE_LEGACY_WORKER_IMPL=disable +NGPU_PER_NODE=1 +ROLLOUT_NAME=vllm + +experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=/code/shared/gsm8k/train.parquet \ + data.val_files=/code/shared/gsm8k/test.parquet \ + data.train_batch_size=${BSZ} \ + data.max_prompt_length=64 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + reward.num_workers=1 \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.model.path=${MODEL} \ + actor_rollout_ref.actor.optim.lr=${LR} \ + actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.strategy=${STRATEGY} \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.n=${ROLL_N} \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.ref.fsdp_config.param_offload=False \ + actor_rollout_ref.ref.strategy=${STRATEGY} \ + algorithm.use_kl_in_reward=False \ + trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ + trainer.critic_warmup=0 \ + trainer.logger=${LOGGER} \ + trainer.experiment_name=${experiment_name} \ + trainer.project_name='verl_arctic_grpo_gsm8k' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_training_steps=${MAX_STEPS} \ + trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log + + # trainer.total_training_steps=${MAX_STEPS} \ diff --git a/verl/experimental/agent_loop/agent_loop.py b/verl/experimental/agent_loop/agent_loop.py index 8879960f128..8e7118db65e 100644 --- a/verl/experimental/agent_loop/agent_loop.py +++ b/verl/experimental/agent_loop/agent_loop.py @@ -918,13 +918,14 @@ def __init__( worker_group: RayWorkerGroup = None, rollout_resource_pool: RayResourcePool = None, reward_loop_worker_handles: list[ray.actor.ActorHandle] = None, - ): + **kwargs, + ): self.config = config self.rollout_config, self.model_config = _get_rollout_and_model_config(config) self.worker_group = worker_group self.rollout_resource_pool = rollout_resource_pool self.reward_loop_worker_handles = reward_loop_worker_handles - + self.kwargs = kwargs assert worker_group is not None or self.rollout_config.nnodes > 0, "nnodes must be > 0 in standalone mode" # for recipe to change @@ -941,9 +942,10 @@ async def create( worker_group: RayWorkerGroup = None, rollout_resource_pool: RayResourcePool = None, reward_loop_worker_handles: list[ray.actor.ActorHandle] = None, + **kwargs, ): """Create agent loop manager.""" - instance = cls(config, worker_group, rollout_resource_pool, reward_loop_worker_handles) + instance = cls(config, worker_group, rollout_resource_pool, reward_loop_worker_handles, **kwargs) await instance._initialize_llm_servers() await instance._init_global_load_balancer() await instance._init_agent_loop_workers() @@ -968,6 +970,7 @@ async def _initialize_llm_servers(self): config=self.rollout_config, model_config=self.model_config, gpus_per_node=self.rollout_config.n_gpus_per_node, + **self.kwargs, ) for replica_rank in range(num_replicas) ] diff --git a/verl/single_controller/ray/base.py b/verl/single_controller/ray/base.py index 2f6ee47064f..a7872b189a1 100644 --- a/verl/single_controller/ray/base.py +++ b/verl/single_controller/ray/base.py @@ -187,8 +187,9 @@ class ResourcePoolManager: resource_pool_spec: dict[str, list[int]] mapping: dict[int, str] resource_pool_dict: dict[str, RayResourcePool] = field(default_factory=dict) + gpu_resource_pool_dict: dict[str, RayResourcePool] = field(default_factory=dict) - def create_resource_pool(self): + def create_resource_pool(self, use_gpu: bool = True): """Create Ray resource pools for distributed training. Initializes resource pools based on the resource pool specification, @@ -202,10 +203,11 @@ def create_resource_pool(self): # For Megatron backend, we recommend using max_colocate_count>1 # that can utilize different WorkerGroup for differnt models resource_pool = RayResourcePool( - process_on_nodes=process_on_nodes, use_gpu=True, max_colocate_count=3, name_prefix=resource_pool_name + process_on_nodes=process_on_nodes, use_gpu=use_gpu, max_colocate_count=3, name_prefix=resource_pool_name ) self.resource_pool_dict[resource_pool_name] = resource_pool - + if use_gpu: + self.gpu_resource_pool_dict[resource_pool_name] = resource_pool self._check_resource_available() def get_resource_pool(self, role) -> RayResourcePool: @@ -214,7 +216,8 @@ def get_resource_pool(self, role) -> RayResourcePool: def get_n_gpus(self) -> int: """Get the number of gpus in this cluster.""" - return sum([n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes]) + process_on_gpu_nodes = [process_on_nodes for pool_name, process_on_nodes in self.resource_pool_spec.items() if pool_name in self.gpu_resource_pool_dict] + return sum([n_gpus for process_on_nodes in process_on_gpu_nodes for n_gpus in process_on_nodes]) def _check_resource_available(self): """Check if the resource pool can be satisfied in this ray cluster.""" @@ -226,9 +229,7 @@ def _check_resource_available(self): # check total required gpus can be satisfied total_available_gpus = sum(node_available_gpus.values()) - total_required_gpus = sum( - [n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes] - ) + total_required_gpus = self.get_n_gpus() if total_available_gpus < total_required_gpus: raise ValueError( f"Total available GPUs {total_available_gpus} is less than total desired GPUs {total_required_gpus}" diff --git a/verl/trainer/config/ppo_trainer.yaml b/verl/trainer/config/ppo_trainer.yaml index fd9b59862ae..2a0779cff1c 100644 --- a/verl/trainer/config/ppo_trainer.yaml +++ b/verl/trainer/config/ppo_trainer.yaml @@ -203,6 +203,9 @@ trainer: # mode: "auto", "enable", or "disable" use_legacy_worker_impl: auto + # whether to use arctic rl + use_arctic_rl: False + # profiler configs global_profiler: diff --git a/verl/trainer/main_ppo.py b/verl/trainer/main_ppo.py index 2c84374d245..262a318be8b 100644 --- a/verl/trainer/main_ppo.py +++ b/verl/trainer/main_ppo.py @@ -134,6 +134,10 @@ def add_actor_rollout_worker(self, config): actor_rollout_cls = ActorRolloutRefWorker ray_worker_group_cls = RayWorkerGroup + if config.trainer.get("use_arctic_rl", False): + from verl.workers.arctic_workers import ActorRolloutRefWorker + actor_rollout_cls = ActorRolloutRefWorker + lora_rank = config.actor_rollout_ref.model.get("lora", {}).get("rank", 0) if lora_rank <= 0: lora_rank = config.actor_rollout_ref.model.get("lora_rank", 0) @@ -340,7 +344,9 @@ def run(self, config): train_sampler = create_rl_sampler(config.data, train_dataset) # Initialize the PPO trainer. - trainer = RayPPOTrainer( + from verl.trainer.ppo.arctic_trainer import ArcticPPOTrainer + ppo_trainer_cls = RayPPOTrainer if not config.trainer.use_arctic_rl else ArcticPPOTrainer + trainer = ppo_trainer_cls( config=config, tokenizer=tokenizer, processor=processor, @@ -356,7 +362,12 @@ def run(self, config): trainer.init_workers() # Start the training process. - trainer.fit() + try: + trainer.fit() + finally: + # Ensure remote services shutdown gracefully + if hasattr(trainer, "destroy"): + trainer.destroy() def create_rl_dataset(data_paths, data_config, tokenizer, processor, is_train=True, max_samples: int = -1): diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py new file mode 100644 index 00000000000..8f7e74b21c9 --- /dev/null +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -0,0 +1,130 @@ +import torch +from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer +from deepspeed.utils import OnDevice +from dss_client.client import DSSInferenceClient, DSSTrainingClient, DSSLogProbClient +import ray +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy +from ray.util.placement_group import placement_group +from verl.workers.rollout.replica import TokenOutput +from tensordict import TensorDict +from typing import Any +from verl.utils.ray_utils import auto_await + +def create_arctic_rl_client(): + sched_pg = placement_group([{"GPU": 0, "CPU": 1}]) + arctic_rl_client = ray.remote( + num_cpus=0, + num_gpus=0, + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=sched_pg, + placement_group_capture_child_tasks=True, + ), + )(ArcticRLClient4VeRL).remote( + ) + + return arctic_rl_client + +def create_meta_model(name_or_path: str): + model_config = AutoConfig.from_pretrained(name_or_path) + with OnDevice(dtype=torch.float16, device='meta'): + meta_model = AutoModelForCausalLM.from_config(model_config) + return meta_model + +class ArcticRLClient4VeRL: + def __init__(self): + self.arctic_inference_client = DSSInferenceClient(dss_server_url="http://localhost:7000") + self.arctic_training_client = DSSTrainingClient(dss_server_url="http://localhost:7000") + self.arctic_log_prob_client = DSSLogProbClient(dss_server_url="http://localhost:7000") + + def initialize(self, model_name: str): + vllm_config = { + "temperature": 0.0, + "top_p": 1.0, + "top_k": 0, + "max_tokens": 1024, + "stop_sequences": [], + "stop_token_ids": [], + } + self.inference_engine = self.arctic_inference_client.initialize( + model_name=model_name, + vllm_config=vllm_config, + ) + self.log_prob_engine = self.arctic_log_prob_client.initialize( + model_name=model_name, + vllm_config=vllm_config, + ) + + ds_config = { + "train_micro_batch_size_per_gpu": 1, + "train_batch_size": 1, + "gradient_accumulation_steps": 1, + "sequence_parallel_size": 1, + "zero_optimization": { + "stage": 1, + }, + } + training_config = { + "optimizer": { + "lr": 0.0002, + "weight_decay": 0.0, + "betas": [0.9, 0.999], + }, + "lr_scheduler": {"warmup_ratio": 0.05}, + "training_horizon": 10, + "max_length": 8096, + "model_config": None, + "attn_implementation": "eager", + } + + self.training_engine = self.arctic_training_client.initialize( + model=create_meta_model(model_name), + ds_config=ds_config, + training_config=training_config) + + self.tokenizer = AutoTokenizer.from_pretrained(model_name) + + def generate(self, prompt_ids, sampling_params) -> TokenOutput: + prompts = [self.tokenizer.decode(prompt_ids)] + return self.inference_engine.generate( + prompts=prompts, + sampling_params=sampling_params, + ) + + def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict): + dss_batch_dict.update(post_process_inputs=post_process_inputs) + + # XXX: somehow we need to differentiate which model is this called on ref vs actor - at the moment it's always actor hardcoded + entropy, log_probs = self.training_engine.fwd_no_grad(**dss_batch_dict) + + # XXX: for some reason no_padding_2_padding expects a 1D tensor - not sure how it'll work for + # bs>1 + # I think it may have to do with tensor.is_nested - different path/logic + # so most likely we need to convert these 2 into TensorDict + if entropy is not None: + # prior_entropy_shape = entropy.shape + entropy = torch.tensor(entropy).squeeze() + if log_probs is not None: + # prior_log_probs_shape = log_probs.shape + log_probs = torch.tensor(log_probs).squeeze() + print(f"arctic_rl_client.compute_log_prob: {entropy.shape=}, {log_probs.shape=}") + return entropy, log_probs + + + def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): + + dss_batch_dict.update(post_process_inputs=post_process_inputs) + + #_ = self.training_engine.forward(**dss_batch_dict, post_process_inputs=post_process_inputs) + _ = self.training_engine.forward(**dss_batch_dict) + loss, metrics = self.training_engine.backward() + self.training_engine.step() + + print(f"arctic_rl_client.update_actor: {loss=}") + print(f"arctic_rl_client.update_actor: {metrics=}") + return loss.cpu().item(), metrics + + def destroy(self): + self.training_engine.destroy() + self.inference_engine.destroy() + return + diff --git a/verl/trainer/ppo/arctic_trainer.py b/verl/trainer/ppo/arctic_trainer.py new file mode 100644 index 00000000000..2d2a4e95f7b --- /dev/null +++ b/verl/trainer/ppo/arctic_trainer.py @@ -0,0 +1,153 @@ +import torch +from typing import Optional +from torch.utils.data import Dataset, Sampler +from verl.trainer.ppo.ray_trainer import RayPPOTrainer +from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup, ResourcePoolManager +from verl.workers.arctic_workers import ActorRolloutRefWorker +from verl.trainer.ppo.utils import Role, WorkerType +from omegaconf import OmegaConf +from verl.single_controller.ray.base import create_colocated_worker_cls +from verl.trainer.ppo.arctic_rl_client import create_arctic_rl_client + +def my_pdb(): + return + import pdb; pdb.set_trace() + +class ArcticPPOTrainer(RayPPOTrainer): + def __init__( + self, + config, + tokenizer, + role_worker_mapping: dict[Role, WorkerType], + resource_pool_manager: ResourcePoolManager, + ray_worker_group_cls: type[RayWorkerGroup] = RayWorkerGroup, + processor=None, + train_dataset: Optional[Dataset] = None, + val_dataset: Optional[Dataset] = None, + collate_fn=None, + train_sampler: Optional[Sampler] = None, + device_name=None, + ): + super().__init__(config=config, + tokenizer=tokenizer, + processor=processor, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + train_dataset=train_dataset, + val_dataset=val_dataset, + collate_fn=collate_fn, + train_sampler=train_sampler, + device_name=device_name) + + self.use_gpu = False + self.rl_client = create_arctic_rl_client() + self.rl_client.initialize.remote(model_name="Qwen/Qwen3-0.6B") + self.wg_kwargs["arctic_rl_client"] = self.rl_client + + + def init_workers(self): + super().init_workers() + return + # print(f"ArcticPPOTrainer.init_workers: {self.actor_rollout_wg=}") + # print(f"ArcticPPOTrainer.init_workers: {self.ref_policy_wg=}") + # print(f"ArcticPPOTrainer.init_workers: {self.async_rollout_manager=}") + # print(f"ArcticPPOTrainer.init_workers: {self.reward_loop_manager=}") + # print(f"ArcticPPOTrainer.init_workers: {self.checkpoint_manager=}") + + # self.resource_pool_manager.create_resource_pool(use_gpu=self.use_gpu) + + # self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} + + # # create actor and rollout + # actor_role = Role.ActorRolloutRef if Role.ActorRolloutRef in self.role_worker_mapping else Role.ActorRollout + # actor_rollout_resource_pool = self.resource_pool_manager.get_resource_pool(actor_role) + # actor_rollout_cls = RayClassWithInitArgs( + # cls=self.role_worker_mapping[actor_role], + # config=self.config.actor_rollout_ref, + # role=str(actor_role), + # ) + # self.resource_pool_to_cls[actor_rollout_resource_pool][str(actor_role)] = actor_rollout_cls + + # # create reference policy if needed + # # if self.use_reference_policy and Role.RefPolicy in self.role_worker_mapping: + # # resource_pool = self.resource_pool_manager.get_resource_pool(Role.RefPolicy) + # # ref_policy_cls = RayClassWithInitArgs( + # # self.role_worker_mapping[Role.RefPolicy], + # # config=self.config.actor_rollout_ref, + # # role=str(Role.RefPolicy), + # # ) + # # self.resource_pool_to_cls[resource_pool][str(Role.RefPolicy)] = ref_policy_cls + + # # initialize WorkerGroup + # # NOTE: if you want to use a different resource pool for each role, which can support different parallel size, + # # you should not use `create_colocated_worker_cls`. + # # Instead, directly pass different resource pool to different worker groups. + # # See https://github.com/volcengine/verl/blob/master/examples/ray/tutorial.ipynb for more information. + # all_wg = {} + # wg_kwargs = {} # Setting up kwargs for RayWorkerGroup + # if OmegaConf.select(self.config.trainer, "ray_wait_register_center_timeout") is not None: + # wg_kwargs["ray_wait_register_center_timeout"] = self.config.trainer.ray_wait_register_center_timeout + # if OmegaConf.select(self.config.global_profiler, "steps") is not None: + # wg_kwargs["profile_steps"] = OmegaConf.select(self.config.global_profiler, "steps") + # # Only require nsight worker options when tool is nsys + # if OmegaConf.select(self.config.global_profiler, "tool") == "nsys": + # assert ( + # OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") + # is not None + # ), "worker_nsight_options must be set when using nsys with profile_steps" + # wg_kwargs["worker_nsight_options"] = OmegaConf.to_container( + # OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") + # ) + # wg_kwargs["device_name"] = self.device_name + + # for resource_pool, class_dict in self.resource_pool_to_cls.items(): + # if not class_dict: + # continue + # worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) + # wg_dict = self.ray_worker_group_cls( + # resource_pool=resource_pool, + # ray_cls_with_init=worker_dict_cls, + # use_gpu=self.use_gpu, + # **wg_kwargs, + # ) + # spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) + # all_wg.update(spawn_wg) + + + # self.actor_rollout_wg = all_wg[str(actor_role)] + # self.actor_rollout_wg.init_model() + + # # create reward loop manager + # from verl.experimental.reward_loop import RewardLoopManager + + # # initalize reward loop manager + # # reward model (colocate or standalone): get resource_pool + # # no reward model: resource_pool = None + # resource_pool = self.resource_pool_manager.get_resource_pool(Role.RewardModel) if self.use_rm else None + # self.reward_loop_manager = RewardLoopManager( + # config=self.config, + # rm_resource_pool=resource_pool, + # ) + + # self.async_rollout_mode = True + # from verl.experimental.agent_loop import AgentLoopManager + + # # enable_agent_reward_loop = not self.use_rm or self.config.reward.reward_model.enable_resource_pool + # # reward_loop_worker_handles = self.reward_loop_manager.reward_loop_workers if enable_agent_reward_loop else None + # # self.async_rollout_manager = AgentLoopManager.create( + # # config=self.config, + # # worker_group=self.actor_rollout_wg, + # # rollout_resource_pool=actor_rollout_resource_pool, + # # reward_loop_worker_handles=reward_loop_worker_handles, + # # ) + + # self.ref_policy_wg = self.actor_rollout_wg + # self.checkpoint_manager = self.actor_rollout_wg + # self.async_rollout_manager = self.actor_rollout_wg + + + + def destroy(self): + # self.actor_rollout_wg.destroy() + self.rl_client.destroy.remote() \ No newline at end of file diff --git a/verl/trainer/ppo/ray_trainer.py b/verl/trainer/ppo/ray_trainer.py index e178ffc143d..478bfd07908 100644 --- a/verl/trainer/ppo/ray_trainer.py +++ b/verl/trainer/ppo/ray_trainer.py @@ -309,6 +309,9 @@ def __init__( self.checkpoint_manager = None + self.wg_kwargs = {} + self.use_gpu = True + def _create_dataloader(self, train_dataset, val_dataset, collate_fn, train_sampler: Optional[Sampler]): """ Creates the train and validation dataloaders. @@ -682,7 +685,7 @@ def init_workers(self): 1. Ray resource pools from configuration 2. Worker groups for each role (actor, critic, etc.) """ - self.resource_pool_manager.create_resource_pool() + self.resource_pool_manager.create_resource_pool(use_gpu=self.use_gpu) self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} @@ -694,6 +697,7 @@ def init_workers(self): cls=self.role_worker_mapping[actor_role], config=self.config.actor_rollout_ref, role=str(actor_role), + **self.wg_kwargs, ) self.resource_pool_to_cls[actor_rollout_resource_pool][str(actor_role)] = actor_rollout_cls else: @@ -840,6 +844,7 @@ def init_workers(self): worker_group=self.actor_rollout_wg, rollout_resource_pool=actor_rollout_resource_pool, reward_loop_worker_handles=reward_loop_worker_handles, + **self.wg_kwargs, ) checkpoint_engine_config = omega_conf_to_dataclass(self.config.actor_rollout_ref.rollout.checkpoint_engine) self.checkpoint_manager = CheckpointEngineManager( @@ -1589,7 +1594,8 @@ def fit(self): metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) # TODO: implement actual tflpo and theoretical tflpo n_gpus = self.resource_pool_manager.get_n_gpus() - metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) + # To support serverless/tinker-like training, we need to support 0 GPUs training + metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=max(n_gpus, 1))) # compute variance proxy metrics gradient_norm = metrics.get("actor/grad_norm", None) metrics.update(compute_variance_proxy_metrics(batch=batch, gradient_norm=gradient_norm)) diff --git a/verl/utils/fsdp_utils.py b/verl/utils/fsdp_utils.py index 8bca54fa88c..11fc247d441 100644 --- a/verl/utils/fsdp_utils.py +++ b/verl/utils/fsdp_utils.py @@ -227,6 +227,19 @@ def load_fsdp_optimizer(optimizer, device_id): state[key] = value.to(device_id, non_blocking=True) +@torch.no_grad() +def get_fsdp_optimizer_devices(optimizer) -> list[torch.device]: + devices = set() + for param_group in optimizer.param_groups: + for param in param_group["params"]: + state = optimizer.state[param] + devices.add(param.device) + for key, value in state.items(): + if isinstance(value, torch.Tensor): + devices.add(value.device) + + return list(devices) + @contextmanager def meta_device_init(): """ diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py new file mode 100644 index 00000000000..0882870c77e --- /dev/null +++ b/verl/workers/arctic_workers.py @@ -0,0 +1,918 @@ +from pathlib import Path +import torch +from verl.utils.ray_utils import auto_await +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.protocol import DataProto +from verl.single_controller.base import Worker +from verl.utils.profiler import DistProfiler, DistProfilerExtension +from verl.workers.engine_workers import ActorRolloutRefWorker as EngineActorRolloutRefWorker +from omegaconf import DictConfig +from tensordict import TensorDict +from dss_client.client import DSSInferenceClient, DSSTrainingClient +from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer +from deepspeed.utils import OnDevice +from verl.utils import tensordict_utils as tu +import os +import ray +from verl.utils.config import omega_conf_to_dataclass +from verl.utils.device import ( + get_device_id, + get_device_name, + get_nccl_backend, + get_torch_device, + set_expandable_segments, +) +from codetiming import Timer +import functools +import logging +import os +from contextlib import nullcontext +from functools import partial +from itertools import chain + +import torch +from codetiming import Timer +from omegaconf import DictConfig, open_dict +from tensordict import NonTensorData, TensorDict +from torch.distributed.device_mesh import init_device_mesh + +try: + from verl.workers.engine.mindspeed.transformer_impl import repatch +except ImportError: + repatch = None +from verl.checkpoint_engine import CheckpointEngineRegistry +from verl.single_controller.base import Worker +from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register +from verl.utils import tensordict_utils as tu +from verl.utils.config import omega_conf_to_dataclass +from verl.utils.device import get_device_name, set_expandable_segments +from verl.utils.distributed import initialize_global_process_group_ray +from verl.utils.flops_counter import FlopsCounter +from verl.utils.memory_utils import aggressive_empty_cache +from verl.utils.metric.utils import Metric +from verl.utils.profiler import DistProfiler, DistProfilerExtension, ProfilerConfig, log_gpu_memory_usage +from verl.utils.py_functional import append_to_dict +from verl.utils.tensordict_utils import maybe_fix_3d_position_ids +from verl.utils.torch_functional import allgather_dict_into_dict +from verl.workers.config import ActorConfig, HFModelConfig, RolloutConfig, TrainingWorkerConfig +from verl.workers.rollout.base import BaseRollout, get_rollout_class +from verl.workers.utils.losses import ppo_loss +from torch import Tensor +from verl.workers.engine.utils import postprocess_batch_func + + + +def create_meta_model(name_or_path: str): + model_config = AutoConfig.from_pretrained(name_or_path) + with OnDevice(dtype=torch.float16, device='meta'): + meta_model = AutoModelForCausalLM.from_config(model_config) + return meta_model + + +DATA_PROTO_KEYS = ["gen_batch_output", "old_log_prob", "ref_log_prob", "compute_advantage", "actor_output"] +TENSOR_DICT_KEYS = ["full_log_prob", "full_ref_log_prob", "full_actor_output"] + +def load_dump_data(train_batch_size, roll_n) -> dict[str, DataProto]: + global_step = 1 + dump_data = {} + dump_path = os.path.join('/code/users/truwase/data/at_verl_dump', f'tbs{train_batch_size}_n{roll_n}') + dump_dir = Path(dump_path) + os.path.exists(dump_dir) + for key in DATA_PROTO_KEYS: + dump_data[key] = DataProto.load_from_disk(Path(dump_dir, f"{global_step}_{key}.pt")) + for key in TENSOR_DICT_KEYS: + dump_data[key] = torch.load(Path(dump_dir, f"{global_step}_{key}.pt"), weights_only=False) + + return dump_data + + +def prepare_model_inputs_remove_padding(micro_batch: TensorDict): + from verl.utils import tensordict_utils as tu + from verl.utils.dataset.dataset_utils import DatasetPadMode + from verl.utils.debug import log_gpu_memory_usage + from verl.utils.device import get_device_id, get_device_name + from verl.utils.model import extract_multi_modal_inputs + from verl.utils.torch_functional import logprobs_from_logits + import verl.utils.torch_functional as verl_F + + use_remove_padding = tu.get_non_tensor_data(data=micro_batch, key="use_remove_padding", default=True) + pad_mode = tu.get_non_tensor_data(data=micro_batch, key="pad_mode", default=DatasetPadMode.NO_PADDING) + use_fused_kernels = tu.get_non_tensor_data(data=micro_batch, key="use_fused_kernels", default=False) + temperature = micro_batch["temperature"] + temperature_item = temperature + if use_fused_kernels: + assert not isinstance(temperature, torch.Tensor), ( + "use_fused_kernels does not support per sample temperature yet" + ) + assert pad_mode == DatasetPadMode.NO_PADDING, f"pad_mode {pad_mode} not supported" + + multi_modal_inputs = extract_multi_modal_inputs(micro_batch.get("multi_modal_inputs", [])) + input_ids = micro_batch["input_ids"] + position_ids = micro_batch["position_ids"] + + if not isinstance(temperature, torch.Tensor): + temperature = torch.tensor([temperature] * input_ids.shape[0], device=input_ids.device) + + temperature = temperature.to(torch.float32) + assert temperature.shape[0] == input_ids.shape[0] + + # args used to get outputs + output_args = {} + + # support per sample temperature + # temperature (bsz,) + # input_ids (bsz, j1) + temperature_rmpad = verl_F.expand_as_nested(temperature, input_ids).values() # (total_nnz,) + temperature_rmpad = temperature_rmpad.unsqueeze(0) # (1, total_nnz) + + if pad_mode == DatasetPadMode.NO_PADDING: + input_ids_rmpad = input_ids.values().unsqueeze(0) # (1, total_nnz) + if position_ids.dim() == 3: + position_ids_rmpad = position_ids.values().unsqueeze(1) # (4, 1, total_nnz) + else: + position_ids_rmpad = position_ids.values().unsqueeze(0) # (1, total_nnz) + else: + raise NotImplementedError(f"pad_mode {pad_mode} not implemented") + + # for compute the log_prob + input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=1) # (1, total_nnz) + + # pad and slice the inputs if sp > 1 + + input_ids_rmpad_rolled = input_ids_rmpad_rolled.squeeze(0) # ((total_nnz / sp) + pad) + temperature_rmpad = temperature_rmpad.squeeze(0) + output_args["input_ids_rmpad_rolled"] = input_ids_rmpad_rolled + output_args["temperature_rmpad"] = temperature_rmpad + + # only pass input_ids and position_ids to enable flash_attn_varlen + + model_inputs = { + "input_ids": input_ids_rmpad, + "attention_mask": None, + "position_ids": position_ids_rmpad, + "labels": input_ids_rmpad, + } + + extra_args = {} + if use_fused_kernels: + extra_args["temperature"] = temperature_item + extra_args["return_dict"] = True + + model_inputs.update(multi_modal_inputs) + model_inputs.update(extra_args) + + return model_inputs, output_args + + + +def prepare_extra_inputs(data: TensorDict) -> dict: + pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) + batch_size = data["input_ids"].shape[0] + seq_len_effective = data["input_ids"].offsets().diff() + max_seq_len = max(seq_len_effective) + ready_input_ids = torch.nested.to_padded_tensor( + data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) + ) + ready_position_ids = torch.nested.to_padded_tensor( + data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) + ) + + extra_inputs = dict( + prompts=data["prompts"], + responses=data["responses"], + attention_mask=data["attention_mask"], + max_response_len=data["max_response_len"], + global_batch_size=data["global_batch_size"], + response_mask=data["response_mask"], + old_log_probs=data["old_log_probs"], + advantages=data["advantages"], + ref_log_prob=data["ref_log_prob"], + rollout_is_weights=data.get("rollout_is_weights", None), + batch_num_tokens=data["loss_mask"].sum(), + ready_input_ids=ready_input_ids, + ready_position_ids=ready_position_ids, + ready_labels=ready_input_ids, + cu_seqlens=data["input_ids"].offsets(), + ) + + + return extra_inputs + +def prepare_log_prob_extra_inputs(data: TensorDict) -> dict: + pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) + batch_size = data["input_ids"].shape[0] + seq_len_effective = data["input_ids"].offsets().diff() + max_seq_len = max(seq_len_effective) + + ready_input_ids = torch.nested.to_padded_tensor( + data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) + ) + ready_position_ids = torch.nested.to_padded_tensor( + data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) + ) + + extra_inputs = dict( + ready_input_ids=ready_input_ids, + ready_position_ids=ready_position_ids, + ready_labels=ready_input_ids, + cu_seqlens=data["input_ids"].offsets() + ) + + return extra_inputs + + +def rm_padding(data: TensorDict, tensor: Tensor) -> Tensor: + cu_seqlens = data["input_ids"].offsets() + seq_lengths = cu_seqlens.diff() # (bsz,) + starts = torch.zeros_like(seq_lengths, dtype=torch.int64) # (bsz,) + tensor = torch.nested.narrow(tensor, 1, starts, seq_lengths, layout=torch.jagged) + tensor = torch.cat([t for t in tensor.unbind()]) + tensor = torch.nested.nested_tensor_from_jagged(tensor, cu_seqlens) + return tensor + +def postprocess_log_prob_output(data: TensorDict, entropy: Tensor, log_probs: Tensor) -> TensorDict: + x_entropy = rm_padding(data, entropy) + x_log_probs = rm_padding(data, log_probs) + + print(f"postprocess_log_prob_output: {x_entropy.shape=} {x_log_probs.shape=} {entropy.shape=} {log_probs.shape=}") + + micro_entropy = [t.unsqueeze(0) for t in x_entropy.unbind()] + micro_log_probs = [t.unsqueeze(0) for t in x_log_probs.unbind()] + output_lst = [] + for i in range(len(micro_entropy)): + model_output = { + "entropy": micro_entropy[i], + "log_probs": micro_log_probs[i], + } + output_lst.append({ + "model_output": model_output, + "metrics": {}, + "loss": 0.0, + }) + + return postprocess_batch_func(output_lst=output_lst, indices=None, data=data) + + +class TrainingWorker(Worker, DistProfilerExtension): + """ + TrainingWorker provides a Tinker-like API (https://thinkingmachines.ai/tinker/) as a RayWorkerGroup + to a single controller. Currently, we only provide more coarse grained APIs, + and do not provide exact APIs as Tinker does. But this can be added in the future. + """ + + def __init__(self, config: TrainingWorkerConfig, actor_config: ActorConfig, arctic_rl_client): + Worker.__init__(self) + + from verl.workers.engine import BaseEngine, EngineRegistry + + #initialize_global_process_group_ray(timeout_second=None) + + self.config = config + self.actor_config = actor_config + + self.arctic_rl_client = arctic_rl_client + + self.model_config = self.config.model_config + self.engine_config = self.config.engine_config + self.optimizer_config = self.config.optimizer_config + self.checkpoint_config = self.config.checkpoint_config + self.device_name = get_device_name() + + if self.engine_config is None: + assert self.optimizer_config is None + if self.config.auto_select_engine_optim_fn is None: + raise ValueError( + "engine_config is not provided and auto_select_engine_optim_fn is not set. " + "Cannot determine engine backend." + ) + # Support automatically select engine backend given model config + self.engine_config, self.optimizer_config = self.config.auto_select_engine_optim_fn( + self.model_config, self.device_name + ) + + # we use the one defined in model + # TODO: this is not elegant and should refactor later + self.engine_config.use_remove_padding = self.model_config.use_remove_padding + self.engine_config.use_fused_kernels = self.model_config.use_fused_kernels + + if repatch is not None: + # NPU MindSpeed patch, will be refactored with MindSpeedEngine. + repatch(self.engine_config.get("override_transformer_config", {})) + + # TODO: add DistProfilerExtension + self.profiler_config = self.config.profiler_config + if self.profiler_config is not None: + self.profiler_tool_config = self.profiler_config.tool_config.get(self.profiler_config.tool, {}) + else: + self.profiler_tool_config = None + + DistProfilerExtension.__init__( + self, DistProfiler(rank=self.rank, config=self.profiler_config, tool_config=self.profiler_tool_config) + ) + + # self.engine: BaseEngine = EngineRegistry.new( + # model_type=self.config.model_type, + # backend=self.engine_config.strategy, + # model_config=self.model_config, + # engine_config=self.engine_config, + # optimizer_config=self.optimizer_config, + # checkpoint_config=self.checkpoint_config, + # ) + + # # build dispatch info + # self._register_dispatch_collect_info( + # mesh_name="train", + # dp_rank=self.engine.get_data_parallel_rank(), + # is_collect=self.engine.is_mp_src_rank_with_outputs(), + # ) + + self.flops_counter = FlopsCounter(self.model_config.hf_config) + + self.loss_fn = None + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def to(self, device, model=True, optimizer=True, grad=True): + """Manual control of load/offload""" + assert device in ["cpu", "device"] + + if device == "device": + device = get_device_name() + + self.engine.to(device=device, model=model, optimizer=optimizer, grad=grad) + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def set_loss_fn(self, loss_fn): + self.loss_fn = loss_fn + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def reset(self): + """ + Reset the model engine to the initial state. If the engine is not initialized, + we initialize it. Otherwise, reload ckpt and reset states + """ + pass # self.engine.initialize() + + def _postprocess_output(self, output, *, global_token_num, delta_time, forward_only, images_seqlens): + """ + + Args: + output: a dictionary containing loss, model_outputs and metrics + + Returns: + + """ + # TODO: whether to log memory + # metrics["perf/max_memory_allocated_gb"] = get_torch_device().max_memory_allocated() / (1024 ** 3) + # metrics["perf/max_memory_reserved_gb"] = get_torch_device().max_memory_reserved() / (1024 ** 3) + # metrics["perf/cpu_memory_used_gb"] = psutil.virtual_memory().used / (1024 ** 3) + + metrics: dict = output.pop("metrics") + # perform all gather in dp group to ensure that it's correct. + # Here each metric in metrics can be a list (micro-batch metrics) or a singleton + # we should always sum the loss of each micro-batch as we scale by global_bsz/global_token + loss = torch.sum(torch.tensor(output.pop("loss"), device=self.device_name)) + + # For grad_norm, we do not perform all reduce because it is already been done when clipping grad + grad_norm = metrics.pop("grad_norm", None) + lr = metrics.pop("lr", None) + + final_metrics = metrics + + final_metrics["loss"] = loss + if grad_norm is not None: + final_metrics["grad_norm"] = grad_norm + if lr is not None: + final_metrics["lr"] = lr + + # TODO: confirm the mtp loss IS same across dp + for k, v in final_metrics.items(): + if k.startswith("mtp_losses"): + flatten_v = [sublist[0] for sublist in v] # sublist should be single element + final_metrics[k] = sum(flatten_v) / len(flatten_v) + # compute mfu + if global_token_num is not None: + estimated_flops, promised_flops = self.flops_counter.estimate_flops( + global_token_num, delta_time, images_seqlens=images_seqlens + ) + final_metrics["mfu"] = estimated_flops / promised_flops + if forward_only: + final_metrics["mfu"] /= 3.0 + # model outputs + model_output = output.pop("model_output", {}) + # We only return final_metrics + final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": final_metrics}) + return final_output + + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train"), blocking=False) + def train_mini_batch(self, data: TensorDict) -> TensorDict: + """Split a batch into N mini-batches run for multiple epochs + + Args: + data: + + Returns: + + """ + batch_size_per_dp = data.shape[0] + disable_auto_offload = tu.pop(data, key="disable_auto_offload", default=False) + mini_batch_size = tu.pop(data, key="mini_batch_size", default=None) + num_mini_batch = tu.pop(data, key="num_mini_batch", default=None) + epochs = tu.pop(data, key="epochs", default=1) + seed = tu.pop(data, key="seed", default=42) + dataloader_kwargs = tu.pop(data, key="dataloader_kwargs", default={}) + + self.engine_config = self.config.engine_config + + assert mini_batch_size is not None or num_mini_batch is not None + + mini_batch_size_per_gpu = mini_batch_size + + # make iterator + dataloader = tu.make_iterator( + data, + mini_batch_size=mini_batch_size_per_gpu, + epochs=epochs, + seed=seed, + dataloader_kwargs=dataloader_kwargs, + ) + + with ( + Timer(name="train_batch", logger=None), + ): + # update + output_lst = [] + total_num_iterations = data.shape[0] // mini_batch_size_per_gpu * epochs + + for batch_idx, mini_batch_td in enumerate(dataloader): + # add global token num + global_token_num = mini_batch_td["input_ids"].offsets().diff().tolist() # (total_nnz,) + tu.assign_non_tensor( + mini_batch_td, + global_token_num=NonTensorData(global_token_num), + update_lr_scheduler=batch_idx == total_num_iterations - 1, + disable_auto_offload=True, + ) + actor_output = self.train_batch(mini_batch_td) + output_lst.append(actor_output) + + actor_output = [tu.get(output, "metrics") for output in output_lst] + metrics = {} + for output in actor_output: + for key, val in output.items(): + print(f"metrics {key=} {val=}") + + # flattn dp and micro batch + if isinstance(val, list): + output[key] = ( + Metric.aggregate_dp(val) + if isinstance(val[0], Metric) + else list(chain.from_iterable(val)) + ) + append_to_dict(metrics, output) + + output = tu.get_tensordict(tensor_dict={}, non_tensor_dict={"metrics": metrics}).cpu() + + return output + + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train"), blocking=False) + def train_batch(self, data: TensorDict) -> TensorDict: + assert self.loss_fn is not None, "loss function can't be None when calling train_batch" + + # global_token_num should be a list of number of tokens of each seq in this batch + global_token_num = tu.get(data, key="global_token_num") + disable_auto_offload = tu.get(data, key="disable_auto_offload", default=False) + images_seqlens = tu.get(data, key="images_seqlens", default=None) + + # inject engineering parameters if not specified + default_keys = dict( + use_remove_padding=self.model_config.use_remove_padding, + use_dynamic_bsz=self.engine_config.use_dynamic_bsz, + max_token_len_per_gpu=self.engine_config.max_token_len_per_gpu, + micro_batch_size_per_gpu=self.engine_config.micro_batch_size_per_gpu, + use_fused_kernels=self.engine_config.use_fused_kernels, + ) + + for key, val in default_keys.items(): + if key not in data.keys(): + tu.assign_non_tensor(data, **{key: val}) + + with ( + Timer(name="train_batch", logger=None) as timer, + ): + # XXX: what's missing is the loss function to be run on the dss side + # arctic-verl/verl/workers/engine/fsdp/transformer_impl.py:1098 forward_step + # the loss function is arctic-verl/verl/workers/utils/losses.py:97 ppo_loss + # from verl.workers.utils.losses import ppo_loss <- need to adapt to pass a gazillion of config variables + + # from verl.utils.tensordict_utils import chunk_tensordict + # batch = chunk_tensordict(data, 1) + print(f"update_actor data: {data}") + + # XXX: fixme + # batch = batch[0] + + dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + print(f"{dss_batch_dict=}") + print(f"{output_args=}") + # import pdb; pdb.set_trace() + + # we need to serialize the config object to dict + # dataclasses.asdict only returns keys that are defined at init (vars will do more) - but perhaps we want `asdict`? + actor_config_as_dict = vars(self.actor_config) + print(f"update_actor: {self.actor_config=}") + print(f"update_actor: {actor_config_as_dict}") + import json + def safe_serialize(obj): + return json.loads(json.dumps(obj, default=lambda o: None)) + #actor_config_as_dict = safe_serialize(self.actor_config) + actor_config_as_dict = safe_serialize(actor_config_as_dict) + + + # pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) + # batch_size = data["input_ids"].shape[0] + # seq_len_effective = data["input_ids"].offsets().diff() + # max_seq_len = max(seq_len_effective) + # ready_input_ids = torch.nested.to_padded_tensor( + # data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) + # ) + # ready_position_ids = torch.nested.to_padded_tensor( + # data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) + # ) + # extra_inputs = dict( + # prompts=data["prompts"], + # responses=data["responses"], + # attention_mask=data["attention_mask"], + # max_response_len=data["max_response_len"], + # global_batch_size=data["global_batch_size"], + # response_mask=data["response_mask"], + # old_log_probs=data["old_log_probs"], + # advantages=data["advantages"], + # ref_log_prob=data["ref_log_prob"], + # rollout_is_weights=data.get("rollout_is_weights", None), + # ready_input_ids=ready_input_ids, + # ready_position_ids=ready_position_ids, + # ready_labels=ready_input_ids, + # # =batch[""], + # ) + # extra_inputs["batch_num_tokens"] = data["loss_mask"].sum() + + extra_inputs = prepare_extra_inputs(data) + policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) + + post_process_inputs = dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config, extra_inputs=extra_inputs) + print(f"update_actor: {post_process_inputs=}") + + # XXX: pass the original batch as post_process_inputs["batch"] - the ppo loss function expects data["prompts"] + # it got stripped and is not in dss_batch_dict +# File "/code/users/stas/github/sf/dss-platform/dss/processors/verl.py", line 90, in fwd_post_process_ppo_loss +# return ppo_loss(config, model_output, data) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "/code/users/stas/github/sf/dss-platform/dss/processors/verl.py", line 100, in ppo_loss +# log_prob = no_padding_2_padding(model_output["log_probs"], data) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "/code/users/stas/github/sf/arctic-verl/verl/workers/utils/padding.py", line 99, in no_padding_2_padding +# prompt_ids = data["prompts"] +# ~~~~^^^^^^^^^^^ +# KeyError: 'prompts' + + loss, metrics = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) + # output = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) + print(f"update_actor: {loss=}") + print(f"update_actor: {metrics=}") + + + from verl.utils.metric import AggregationType, Metric + # XXX: fix me - we need to aggregate the metrics + metrics = {k:Metric(value=v[0], aggregation=AggregationType.MEAN) for k,v in metrics.items()} + + delta_time = timer.last + + # XXX: fix me + # metrics = { + # 'actor/pg_clipfrac': None, + # 'actor/ppo_kl': None, + # 'actor/pg_clipfrac_lower': None, + # 'actor/pg_loss': None, + # 'kl_loss': None, + # 'kl_coef': None, + # 'grad_norm': None, + # } + + print(f"{data=}") + print(f"{data["input_ids"].shape=}") + model_output = { + # XXX: fix me - made a copy of existing same shape tensor for now + # 'log_probs': batch[0]["ref_log_prob"] + } + + # expected output so far + # + # output={ + # 'model_output': { + # 'log_probs': NestedTensor(size=(1,j18), offsets=tensor([ 0,401], device='cuda:0'), grad_fn=, contiguous=True) + # }, + # 'loss': [-0.9999991059303284], + # 'metrics': { + # 'actor/pg_clipfrac': , + # 'actor/ppo_kl': , + # 'actor/pg_clipfrac_lower': , + # 'actor/pg_loss': , + # 'kl_loss': , + # 'kl_coef': [0.001], + # 'grad_norm': 16.321151733398438, + # } + # } + + + #output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict=non_tensor_dict) + output = dict( + model_output=model_output, + metrics=metrics, + loss=loss, + ) + + update_lr_scheduler = tu.get(data, key="update_lr_scheduler", default=False) + # XXX: fix me + update_lr_scheduler = False + # update lr scheduler + if update_lr_scheduler: + lr = self.engine.lr_scheduler_step() + else: + lr = None + + + # we don't need model_output in training. Maybe we change out mind later + #output.pop("model_output") + if lr is not None: + output["metrics"]["lr"] = lr + + final_output = self._postprocess_output( + output, + global_token_num=global_token_num, + delta_time=delta_time, + forward_only=False, + images_seqlens=images_seqlens, + ).cpu() + + return final_output + + + + + + +class ActorRolloutRefWorker(Worker, DistProfilerExtension): + def __init__(self, config: DictConfig, role: str, **kwargs): + Worker.__init__(self) + self.config = config + self.role = role + self._is_actor = self.role in ["actor", "actor_rollout", "actor_rollout_ref"] + self._is_rollout = self.role in ["rollout", "actor_rollout", "actor_rollout_ref"] + self._is_ref = self.role in ["ref", "actor_rollout_ref"] + + self.arctic_rl_client = kwargs.get("arctic_rl_client", None) + + # assert self.arctic_rl_client is not None, "arctic_rl_client is required" + self._loaded_dump_data = load_dump_data(1, 1) + DistProfilerExtension.__init__(self, DistProfiler(rank=self.rank, config=None, tool_config=None)) + + if self._is_actor: + model_config: HFModelConfig = omega_conf_to_dataclass(self.config.model) + actor_config: ActorConfig = omega_conf_to_dataclass(self.config.actor) + actor_config.model_config = model_config + actor_training_config = TrainingWorkerConfig( + model_type="language_model", + model_config=actor_config.model_config, + engine_config=actor_config.engine, + optimizer_config=actor_config.optim, + checkpoint_config=actor_config.checkpoint, + ) + self.actor_config = actor_config + + assert self.config.actor.use_dynamic_bsz == self.config.rollout.log_prob_use_dynamic_bsz + + # assign engine configs + actor_training_config.engine_config.use_dynamic_bsz = self.config.actor.use_dynamic_bsz + actor_training_config.engine_config.infer_max_token_len_per_gpu = ( + self.config.rollout.log_prob_max_token_len_per_gpu + ) + actor_training_config.engine_config.infer_micro_batch_size_per_gpu = ( + self.config.rollout.log_prob_micro_batch_size_per_gpu + ) + actor_training_config.engine_config.max_token_len_per_gpu = self.config.actor.ppo_max_token_len_per_gpu + actor_training_config.engine_config.micro_batch_size_per_gpu = ( + self.config.actor.ppo_micro_batch_size_per_gpu + ) + actor_training_config.engine_config.use_remove_padding = model_config.use_remove_padding + + if self.config.actor.use_dynamic_bsz: + assert self.config.rollout.log_prob_max_token_len_per_gpu is not None + assert self.config.actor.ppo_max_token_len_per_gpu is not None + else: + assert self.config.rollout.log_prob_micro_batch_size_per_gpu is not None + assert self.config.actor.ppo_micro_batch_size_per_gpu is not None + + self.actor = TrainingWorker(config=actor_training_config, actor_config=actor_config, arctic_rl_client=self.arctic_rl_client) + + self.actor.reset() + self.loss_fn = partial(ppo_loss, config=actor_config) + self.actor.set_loss_fn(loss_fn=self.loss_fn) + + self.set_dispatch_collect(mesh_name="actor", **self.actor.get_dispatch_collect()) + + # from verl.workers.actor import DataParallelPPOActor + + # # hacks to appease to DataParallelPPOActor + # import torch.distributed + # torch.distributed.get_rank = lambda: 0 + + # actor_cfg = omega_conf_to_dataclass(self.config.actor) + # self.actor = DataParallelPPOActor( + # # XXX: hijack actor_module + # config=actor_cfg, actor_module=None, actor_optimizer=None + # ) + + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def init_model(self): + self._register_dispatch_collect_info("actor", dp_rank=self.rank, is_collect=True) + self._register_dispatch_collect_info("ref", dp_rank=self.rank, is_collect=True) + self._register_dispatch_collect_info("rollout", dp_rank=self.rank, is_collect=True) + + return + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def destroy(self): + self.dss_training_engine.destroy() + self.arctic_inference_engine.destroy() + return + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def set_loss_fn(self, loss_fn): + return + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def to(self, device, model=True, optimizer=True, grad=True): + """Manual control of load/offload""" + return + + + def _update_config_params(self, data: TensorDict): + default_keys = dict( + use_remove_padding=self.actor.model_config.use_remove_padding, + use_dynamic_bsz=self.actor.engine_config.use_dynamic_bsz, + max_token_len_per_gpu=self.actor.engine_config.max_token_len_per_gpu, + micro_batch_size_per_gpu=self.actor.engine_config.micro_batch_size_per_gpu, + use_fused_kernels=self.actor.engine_config.use_fused_kernels, + ) + + for key, val in default_keys.items(): + if key not in data.keys(): + tu.assign_non_tensor(data, **{key: val}) + + + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="ref")) + @DistProfiler.annotate(color="olive", role="ref_compute_log_prob") + def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: + # return self._loaded_dump_data["full_ref_log_prob"] + # import pdb; pdb.set_trace() + # from verl.utils.tensordict_utils import chunk_tensordict + # batch = chunk_tensordict(data, 1) + # print(f"compute_ref_log_prob data: {data}") + dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + # print(f"{dss_batch_dict=}") + # import pdb; pdb.set_trace() + # self.dss_training_engine.forward(**dss_batch_dict) + # loss = self.dss_training_engine.backward() + # print(f"loss: {loss}") + # import pdb; pdb.set_trace() + # log_prob = self._loaded_dump_data["full_log_prob"] + + self._update_config_params(data) + post_process_inputs = prepare_log_prob_extra_inputs(data) + entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) + + batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) + model_output = batch_output.pop("model_output", {}) + metrics = { + "mfu": 0.0, + "loss": 1.0, + } + final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) + + + # metrics = { + # "mfu": 0.0, + # "loss": 1.0, + # "batch_size": 1, + # } + + # model_output = { + # "log_probs": log_probs, + # } + + # final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) + + return final_output + + + + + # TODO: Actor API Begin + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") + def compute_log_prob(self, data: TensorDict) -> TensorDict: + # import pdb; pdb.set_trace() + # from verl.utils.tensordict_utils import chunk_tensordict + # batch = chunk_tensordict(data, 1) + # print(f"compute_log_prob data: {data}") + dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + # print(f"{dss_batch_dict=}") + # import pdb; pdb.set_trace() + # self.dss_training_engine.forward(**dss_batch_dict) + # loss = self.dss_training_engine.backward() + # print(f"loss: {loss}") + # import pdb; pdb.set_trace() + # log_prob = self._loaded_dump_data["full_log_prob"] + + self._update_config_params(data) + post_process_inputs = prepare_log_prob_extra_inputs(data) + # print(f"compute_log_prob: {post_process_inputs=}") + + entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) + + # import pdb; pdb.set_trace() + batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) + model_output = batch_output.pop("model_output", {}) + metrics = { + "mfu": 0.0, + "loss": 1.0, + } + final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) + + return final_output + + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + @DistProfiler.annotate(color="red", role="actor_update") + def update_actor(self, data: TensorDict) -> TensorDict: + output = self.actor.train_mini_batch(data=data) + return output.cpu() if output is not None else None + + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def load_checkpoint(self, local_path, hdfs_path=None, del_local_after_load=False): + assert "actor" in self.role, "load_checkpoint only support actor role" + return + + + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): + assert "actor" in self.role, "save_checkpoint only support actor role" + return + + @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) + async def update_weights(self, global_steps: int = None): + """Update weights from trainer to rollout. + + 1. For sync training with colocated trainer and rollout, update rollout directly from model engine. + - before update_weights: rollout should be in sleep mode. + - after update_weights: rollout should be in wake_up mode. + 2. For async training with disaggregated trainer and rollout, send_weights only by checkpoint engine. + """ + return + + # TODO: Actor API End + + + # TODO: Rollout API Begin + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="rollout")) + async def generate_sequences(self, batch: DataProto): + print(f"{batch.non_tensor_batch=}") + raw_prompts = list(batch.non_tensor_batch["raw_prompt"]) + print(f"{raw_prompts=}") + prompts = self.tokenizer.apply_chat_template( + raw_prompts, + add_generation_prompt=True, + tokenize=False, + ) + # import pdb; pdb.set_trace() + print(f"prompts: {prompts}") + metrics = {} + + gen_batch_output = self.arctic_inference_engine.generate(prompts=prompts) + + return gen_batch_output + # return self._loaded_dump_data["gen_batch_output"] + + # TODO: Rollout API End + + # TODO: CheckpointManager API Begin + @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) + async def sleep_replicas(self): + """Sleep all rollout replicas: free weight and kv_cache device memory.""" + return + # TODO: CheckpointManager API \ No newline at end of file diff --git a/verl/workers/engine_workers.py b/verl/workers/engine_workers.py index d0c065e4dfd..5367479a5bf 100644 --- a/verl/workers/engine_workers.py +++ b/verl/workers/engine_workers.py @@ -667,7 +667,7 @@ async def update_weights(self, global_steps: int = None): log_gpu_memory_usage("After update_weights", logger=logger) # 3. offload model to cpu - self.actor.engine.to("cpu", model=True, optimizer=False, grad=False) + self.actor.engine.to("cpu", model=self.actor.engine.is_param_offload_enabled, optimizer=False, grad=False) aggressive_empty_cache(force_sync=True) # 4. resume kv_cache diff --git a/verl/workers/rollout/arctic_rollout/__init__.py b/verl/workers/rollout/arctic_rollout/__init__.py new file mode 100644 index 00000000000..cf453f8b2f6 --- /dev/null +++ b/verl/workers/rollout/arctic_rollout/__init__.py @@ -0,0 +1,3 @@ +from .arctic_rollout import ArcticReplica + +__all__ = ["ArcticReplica"] diff --git a/verl/workers/rollout/arctic_rollout/arctic_rollout.py b/verl/workers/rollout/arctic_rollout/arctic_rollout.py new file mode 100644 index 00000000000..c744b4a00fd --- /dev/null +++ b/verl/workers/rollout/arctic_rollout/arctic_rollout.py @@ -0,0 +1,322 @@ +import ray +from typing import Any, Optional +from verl.workers.rollout.vllm_rollout.vllm_async_server import vLLMHttpServer + +import argparse +from typing import Any, Optional +from verl.trainer.ppo.arctic_rl_client import ArcticRLClient4VeRL +from collections.abc import AsyncGenerator + +import ray +from ray.actor import ActorHandle +from vllm import SamplingParams +from vllm.inputs import TokensPrompt +from vllm.lora.request import LoRARequest +from vllm.outputs import RequestOutput, CompletionOutput + +from verl.utils.tokenizer import normalize_token_ids +from verl.workers.config import HFModelConfig, RolloutConfig +from verl.workers.rollout.replica import RolloutMode, RolloutReplica, TokenOutput +from verl.workers.rollout.vllm_rollout.utils import ( + VLLM_LORA_INT_ID, + VLLM_LORA_NAME, + VLLM_LORA_PATH, +) +from transformers import AutoTokenizer +from verl.utils.config import omega_conf_to_dataclass +from verl.workers.rollout.utils import get_max_position_embeddings + + +class ArcticLLMEngine: + def __init__( + self, + replica_rank: int, + arctic_rl_client: ArcticRLClient4VeRL, + ): + self.replica_rank = replica_rank + self.arctic_rl_client = arctic_rl_client + self.tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") + + async def generate( + self, + prompt: TokensPrompt, + sampling_params: dict[str, Any], + request_id: str, + lora_request: Optional[LoRARequest] = None, + priority: int = 0, + ) -> AsyncGenerator[RequestOutput, None]: + gen_batch_output = await self.arctic_rl_client.generate.remote( + prompt_ids=prompt['prompt_token_ids'], + sampling_params=sampling_params, + ) + # print(f"arctic_async_server: {gen_batch_output=}, {type(gen_batch_output)=}") + # gen_batch_output = await ray.get(gen_batch_output) + + raw_prompt = self.tokenizer.decode(prompt['prompt_token_ids']) + completed_outputs = [] + for i, output in enumerate(gen_batch_output): + completed_outputs.append(CompletionOutput( + index=i, + text=output['text'], + token_ids=output['token_ids'], + finish_reason=output['finish_reason'], + cumulative_logprob=None, + logprobs=None + ) + ) + + yield RequestOutput( + request_id=request_id, + outputs=completed_outputs, + prompt=raw_prompt, + prompt_logprobs=None, + prompt_token_ids=prompt['prompt_token_ids'], + # finished=completed_output.finish_reason == "stop", + finished=True, + ) + + +class ArcticLLMServer(vLLMHttpServer): + """vLLM http server in single node, this is equivalent to launch server with command line: + ``` + vllm serve --tensor-parallel-size=8 ... + ``` + """ + + def __init__( + self, + config: RolloutConfig, + model_config: HFModelConfig, + rollout_mode: RolloutMode, + arctic_rl_client: ArcticRLClient4VeRL, + workers: list[ActorHandle] = [], + replica_rank: int = 0, + node_rank: int = 0, + gpus_per_node: int = 1, + nnodes: int = 1, + cuda_visible_devices: str = "0", + ): + """ + Args: + config (RolloutConfig): full config. + model_config (HFModelConfig): model config. + rollout_mode (RolloutMode): rollout mode. + replica_rank (int): replica rank, a replica may contain multiple nodes. + node_rank (int): node rank. + gpus_per_node (int): number of gpus per node. + nnodes (int): number of nodes. + cuda_visible_devices (str): cuda visible devices. + """ + self.config: RolloutConfig = omega_conf_to_dataclass(config) + self.model_config: HFModelConfig = omega_conf_to_dataclass(model_config, dataclass_type=HFModelConfig) + max_position_embeddings = get_max_position_embeddings(self.model_config.hf_config) + if self.config.max_model_len is None: + self.config.max_model_len = max_position_embeddings + else: + if self.config.max_model_len > max_position_embeddings: + raise ValueError( + f"max_model_len ({self.config.max_model_len}) should be less than or equal to " + f"max_position_embeddings ({max_position_embeddings})" + ) + + self.rollout_mode = rollout_mode + self.workers = workers + + self.replica_rank = replica_rank + self.node_rank = node_rank + self.gpus_per_node = gpus_per_node + self.nnodes = nnodes + # model weights version, set by ServerAdapter when update weights. + self.global_steps = None + + if self.rollout_mode != RolloutMode.HYBRID and self.config.load_format == "dummy": + # logger.warning(f"rollout mode is {self.rollout_mode}, load_format is dummy, set to auto") + self.config.load_format = "auto" + + + self._master_address = None + self._master_port = None + self._dp_rpc_port = None + self._dp_master_port = None + + self.engine = ArcticLLMEngine(replica_rank, arctic_rl_client) + + # logger.info( + # f"vLLMHttpServer, replica_rank: {self.replica_rank}, node_rank: {self.node_rank}, " + # f"{get_visible_devices_keyword()}: {cuda_visible_devices}, " + # f"master_address: {self._master_address}, master_port: {self._master_port}, " + # f"data_parallel_rpc_port: {self._dp_rpc_port}, data_parallel_master_port: {self._dp_master_port}" + # ) + + def get_master_address(self): pass + + def get_server_address(self): pass + + @property + def lora_as_adapter(self) -> bool: pass + + async def collective_rpc( + self, + **kwargs, + ): + pass + + async def launch_server(self, master_address: str = None, master_port: int = None, dp_rpc_port: int = None): + pass + + async def run_server(self, args: argparse.Namespace): + pass + + + async def generate( + self, + prompt_ids: list[int], + sampling_params: dict[str, Any], + request_id: str, + image_data: Optional[list[Any]] = None, + video_data: Optional[list[Any]] = None, + priority: int = 0, + ) -> TokenOutput: + """Generate sequence with token-in-token-out.""" + prompt_ids = normalize_token_ids(prompt_ids) + + # Calculate the maximum possible new tokens based on available context space + # This serves as a safety upper bound + max_possible_tokens = self.config.max_model_len - len(prompt_ids) + if max_possible_tokens < 0: + raise ValueError( + f"Prompt length ({len(prompt_ids)}) exceeds the model's maximum context length " + f"({self.config.max_model_len})." + ) + + # Determine max_tokens from sampling_params or use configured response_length as default + if "max_tokens" in sampling_params: + max_tokens = sampling_params.pop("max_tokens") + elif "max_new_tokens" in sampling_params: + # support sglang-style 'max_new_tokens' param + max_tokens = sampling_params.pop("max_new_tokens") + else: + # Default to a calculation that considers configured lengths + max_tokens = self.config.response_length + self.config.prompt_length - len(prompt_ids) + + # Clamp max_tokens to the valid range [0, max_possible_tokens] + max_tokens = max(0, min(max_tokens, max_possible_tokens)) + + assert max_tokens <= max_possible_tokens, ( + f"max_tokens {max_tokens} exceeds available context space {max_possible_tokens}" + ) + sampling_params["logprobs"] = 0 if sampling_params.pop("logprobs", False) else None + sampling_params.setdefault("repetition_penalty", self.config.get("repetition_penalty", 1.0)) + # sampling_params = SamplingParams(max_tokens=max_tokens, **sampling_params) + sampling_params["max_tokens"] = max_tokens + multi_modal_data = {} + if image_data is not None: + multi_modal_data["image"] = image_data + if video_data is not None: + multi_modal_data["video"] = video_data + # import pdb; pdb.set_trace() + prompt = TokensPrompt(prompt_token_ids=prompt_ids, multi_modal_data=multi_modal_data) + + # Add lora request + lora_request = None + if self.lora_as_adapter: + # Make sure we also check that the lora is already loaded in the engine + lora_loaded = VLLM_LORA_INT_ID in await self.engine.list_loras() + if lora_loaded: + lora_request = LoRARequest( + lora_name=VLLM_LORA_NAME, lora_int_id=VLLM_LORA_INT_ID, lora_path=VLLM_LORA_PATH + ) + # import pdb; pdb.set_trace() + generator = self.engine.generate( + prompt=prompt, + sampling_params=sampling_params, + request_id=request_id, + lora_request=lora_request, + priority=priority, + ) + + # print(f"arctic_async_server: {generator=}, {type(generator)=}") + + # Get final response + final_res: Optional[RequestOutput] = None + async for output in generator: + final_res = output + assert final_res is not None + + token_ids = final_res.outputs[0].token_ids + log_probs = None + if sampling_params["logprobs"] is not None: + log_probs = [logprobs[token_ids[i]].logprob for i, logprobs in enumerate(final_res.outputs[0].logprobs)] + + routed_experts = None + if self.config.enable_rollout_routing_replay: + routed_experts = final_res.outputs[0].routed_experts + + # Determine stop reason from finish_reason + finish_reason = final_res.outputs[0].finish_reason + if finish_reason == "abort": + stop_reason = "aborted" + elif finish_reason in ("stop", "length"): + stop_reason = "completed" + else: + stop_reason = finish_reason # for more stop reason in the future + + num_preempted = None + + if hasattr(final_res.outputs[0], "num_preempted"): + num_preempted = final_res.outputs[0].num_preempted + + return TokenOutput( + token_ids=token_ids, + log_probs=log_probs, + routed_experts=routed_experts, + stop_reason=stop_reason, + num_preempted=num_preempted, + extra_info={"global_steps": self.global_steps}, + ) + + + + +class ArcticReplica(RolloutReplica): + def __init__( + self, + replica_rank: int, + config: RolloutConfig, + model_config: HFModelConfig, + gpus_per_node: int = 1, + is_reward_model: bool = False, + **kwargs, + ): + super().__init__(replica_rank, config, model_config, gpus_per_node, is_reward_model) + self.server_class = ray.remote(ArcticLLMServer) + self.arctic_rl_client = kwargs.get("arctic_rl_client", None) + # assert self.arctic_rl_client is not None, "arctic_rl_client is required" + + + def rollout_worker_use_gpu(self) -> bool: + return False + + + async def launch_servers(self): + server = self.server_class.options( + ).remote( + replica_rank=self.replica_rank, + config=self.config, + model_config=self.model_config, + rollout_mode=self.rollout_mode, + arctic_rl_client=self.arctic_rl_client, + ) + self.servers.append(server) + self._server_handle = server + + + async def wake_up(self): + pass + + async def sleep(self): + pass + + async def abort_request(self, request_id: str) -> dict[str, Any]: + return {"aborted": True, "request_id": 0} + \ No newline at end of file diff --git a/verl/workers/rollout/replica.py b/verl/workers/rollout/replica.py index 969c6208083..2557eb74d7a 100644 --- a/verl/workers/rollout/replica.py +++ b/verl/workers/rollout/replica.py @@ -348,11 +348,16 @@ def _load_trtllm(): return TRTLLMReplica +def _load_arctic(): + from verl.workers.rollout.arctic_rollout.arctic_rollout import ArcticReplica + + return ArcticReplica # Register built-in types RolloutReplicaRegistry.register("vllm", _load_vllm) RolloutReplicaRegistry.register("sglang", _load_sglang) RolloutReplicaRegistry.register("trtllm", _load_trtllm) +RolloutReplicaRegistry.register("arctic", _load_arctic) # Original function for backward compatibility From 81f98376730c169a82138e9a3400ea5edec42837 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 2 Apr 2026 17:53:18 +0000 Subject: [PATCH 05/58] Disable CI --- .github/workflows_old/README.md | 73 ++++ .github/workflows_old/check-pr-title.yml | 58 ++++ .github/workflows_old/cpu_unit_tests.yml | 118 +++++++ .github/workflows_old/doc.yml | 101 ++++++ .../workflows_old/docker-build-ascend-a2.yml | 84 +++++ .../workflows_old/docker-build-ascend-a3.yml | 84 +++++ .github/workflows_old/e2e_ascend.yml | 166 +++++++++ .../workflows_old/e2e_fully_async_policy.yml | 170 ++++++++++ .../workflows_old/e2e_one_step_off_policy.yml | 171 ++++++++++ .../e2e_one_step_off_policy_ascend.yml | 169 ++++++++++ .../e2e_ppo_grpo_trainer_trtllm.yml | 287 ++++++++++++++++ .github/workflows_old/e2e_ppo_trainer.yml | 78 +++++ .../e2e_ppo_trainer_megatron_sglang.yml | 201 +++++++++++ .../e2e_ppo_trainer_megatron_sglang_2.yml | 201 +++++++++++ .../e2e_ppo_trainer_megatron_vllm.yml | 212 ++++++++++++ .../e2e_ppo_trainer_megatron_vllm_2.yml | 318 ++++++++++++++++++ ...e2e_ppo_trainer_megatron_vllm_2_ascend.yml | 233 +++++++++++++ .../e2e_ppo_trainer_veomni_vllm.yml | 153 +++++++++ .github/workflows_old/e2e_sft_llm.yml | 153 +++++++++ .github/workflows_old/e2e_sft_llm_ascend.yml | 160 +++++++++ .github/workflows_old/e2e_sft_vlm.yml | 128 +++++++ .github/workflows_old/gpu_unit_tests.yml | 137 ++++++++ .github/workflows_old/model.yml | 184 ++++++++++ .github/workflows_old/model_ascend.yml | 137 ++++++++ .github/workflows_old/nightly_ascend.yml | 174 ++++++++++ .github/workflows_old/npu_unit_tests.yml | 126 +++++++ .github/workflows_old/pre-commit.yml | 41 +++ .github/workflows_old/precommit-autofix.yml | 52 +++ .github/workflows_old/reward_model_sglang.yml | 134 ++++++++ .github/workflows_old/reward_model_vllm.yml | 134 ++++++++ .../reward_model_vllm_ascend.yml | 113 +++++++ .github/workflows_old/sanity.yml | 108 ++++++ .github/workflows_old/scorecard.yml | 66 ++++ .github/workflows_old/secrets_scan.yml | 22 ++ .github/workflows_old/sgl.yml | 165 +++++++++ .github/workflows_old/type-coverage-check.yml | 31 ++ .github/workflows_old/vllm.yml | 169 ++++++++++ 37 files changed, 5111 insertions(+) create mode 100644 .github/workflows_old/README.md create mode 100644 .github/workflows_old/check-pr-title.yml create mode 100644 .github/workflows_old/cpu_unit_tests.yml create mode 100644 .github/workflows_old/doc.yml create mode 100644 .github/workflows_old/docker-build-ascend-a2.yml create mode 100644 .github/workflows_old/docker-build-ascend-a3.yml create mode 100644 .github/workflows_old/e2e_ascend.yml create mode 100644 .github/workflows_old/e2e_fully_async_policy.yml create mode 100644 .github/workflows_old/e2e_one_step_off_policy.yml create mode 100644 .github/workflows_old/e2e_one_step_off_policy_ascend.yml create mode 100644 .github/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml create mode 100644 .github/workflows_old/e2e_ppo_trainer.yml create mode 100644 .github/workflows_old/e2e_ppo_trainer_megatron_sglang.yml create mode 100644 .github/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml create mode 100644 .github/workflows_old/e2e_ppo_trainer_megatron_vllm.yml create mode 100644 .github/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml create mode 100644 .github/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml create mode 100644 .github/workflows_old/e2e_ppo_trainer_veomni_vllm.yml create mode 100644 .github/workflows_old/e2e_sft_llm.yml create mode 100644 .github/workflows_old/e2e_sft_llm_ascend.yml create mode 100644 .github/workflows_old/e2e_sft_vlm.yml create mode 100644 .github/workflows_old/gpu_unit_tests.yml create mode 100644 .github/workflows_old/model.yml create mode 100644 .github/workflows_old/model_ascend.yml create mode 100644 .github/workflows_old/nightly_ascend.yml create mode 100644 .github/workflows_old/npu_unit_tests.yml create mode 100644 .github/workflows_old/pre-commit.yml create mode 100644 .github/workflows_old/precommit-autofix.yml create mode 100644 .github/workflows_old/reward_model_sglang.yml create mode 100644 .github/workflows_old/reward_model_vllm.yml create mode 100644 .github/workflows_old/reward_model_vllm_ascend.yml create mode 100644 .github/workflows_old/sanity.yml create mode 100644 .github/workflows_old/scorecard.yml create mode 100644 .github/workflows_old/secrets_scan.yml create mode 100644 .github/workflows_old/sgl.yml create mode 100644 .github/workflows_old/type-coverage-check.yml create mode 100644 .github/workflows_old/vllm.yml diff --git a/.github/workflows_old/README.md b/.github/workflows_old/README.md new file mode 100644 index 00000000000..d83c87b2e71 --- /dev/null +++ b/.github/workflows_old/README.md @@ -0,0 +1,73 @@ +### Adding a New Workflow + +When adding a new workflow for continuous integration (CI), you have two runner options: a fixed runner or a machine from the vemlp. + +- **Fixed Runner**: To use a fixed runner, specify it in your workflow using the `runs-on` keyword, like `runs-on: [L20x8]`. +- **Vemlp Runner**: Opting for a Vemlp machine allows you to launch tasks elastically. + +Here is a template to assist you. This template is designed for using Vemlp machines. Currently, for each workflow, you need to create a `setup` and a `cleanup` job. When using this template, the main parts you need to modify are the `IMAGE` environment variable and the specific `job steps`. + +```yaml +name: Your Default Workflow + +on: + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - ".github/workflows/template.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + +env: + IMAGE: "your vemlp image" # e.g. "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" + DYNAMIC_RUNNER_URL: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" # public veFaas api + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + task-id: ${{ steps.create-runner.outputs.task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_URL }}" + image: "${{ env.DEFAULT_IMAGE }}" + + your_job: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'default-runner' }}"] + steps: + xxxx # your jobs + + cleanup: + runs-on: ubuntu-latest + needs: [setup, your_job] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_URL }}" + task-id: "${{ needs.setup.outputs.task-id }}" +``` + +### Model and Dataset +To avoid CI relies on network, we pre-download dataset on a NFS on the CI machine. The path for models are \${HOME}/models and the path for dataset is \${HOME}/models/hf_data. \ No newline at end of file diff --git a/.github/workflows_old/check-pr-title.yml b/.github/workflows_old/check-pr-title.yml new file mode 100644 index 00000000000..948ce5e3f01 --- /dev/null +++ b/.github/workflows_old/check-pr-title.yml @@ -0,0 +1,58 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + + +on: + pull_request: + types: [opened, edited, synchronize] + +jobs: + check-title: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Run PR title checker + run: python3 tests/special_sanity/check_pr_title.py + env: + PR_TITLE: ${{ github.event.pull_request.title }} + + - name: Run PR description checker + run: python3 tests/special_sanity/check_pr_description.py + env: + PR_TITLE: ${{ github.event.pull_request.title }} + GITHUB_EVENT_PATH: ${{ github.event_path }} diff --git a/.github/workflows_old/cpu_unit_tests.yml b/.github/workflows_old/cpu_unit_tests.yml new file mode 100644 index 00000000000..48ce123bc07 --- /dev/null +++ b/.github/workflows_old/cpu_unit_tests.yml @@ -0,0 +1,118 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: cpu_unit_tests + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - .github/workflows/cpu_unit_tests.yml + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + cpu_unit_tests: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + TORCH_COMPILE_DISABLE: 1 + TORCHINDUCTOR_DISABLE: 1 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install --upgrade "transformers>=5.0.0" + - name: Download datasets + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k + - name: Running CPU unit tests + run: | + echo '[pytest]' > pytest.ini + echo 'python_files = *_on_cpu.py' >> pytest.ini + pytest -s -x --asyncio-mode=auto tests/ + cleanup: + runs-on: ubuntu-latest + needs: [setup, cpu_unit_tests] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/doc.yml b/.github/workflows_old/doc.yml new file mode 100644 index 00000000000..aa4a713deac --- /dev/null +++ b/.github/workflows_old/doc.yml @@ -0,0 +1,101 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + + +name: doc_test + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "docs/**" + - .github/workflows/doc.yml + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read # for checkout + pages: write # for deploy-pages + id-token: write # for deploy-pages + +jobs: + doc_test: + runs-on: ubuntu-latest + timeout-minutes: 5 # Increase this timeout value as needed + strategy: + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip install -r docs/requirements-docs.txt + + - name: Run doc make html + run: | + cd docs + make clean + make html SPHINXOPTS="--keep-going -w _build/sphinx.log" + if grep -q ": ERROR:" _build/sphinx.log; then + echo "🚨 Sphinx doc build contained ERRORs - see _build/sphinx.log" + exit 1 + fi + if grep -q "WARNING: document isn't included in any toctree" _build/sphinx.log; then + echo "🚨 Sphinx doc build contained WARNING. Please include newly added docs in index.rst. See _build/sphinx.log for details" + exit 1 + fi + if grep -q "WARNING: Inline emphasis" _build/sphinx.log; then + echo "🚨 Sphinx doc build contained WARNING. Please check inline emphasis is correct. See _build/sphinx.log for details" + exit 1 + fi + if grep -q "WARNING: Definition list ends without a blank line" _build/sphinx.log; then + echo "🚨 Sphinx doc build contained WARNING. Please check if the indentation is correct. See _build/sphinx.log for details" + exit 1 + fi diff --git a/.github/workflows_old/docker-build-ascend-a2.yml b/.github/workflows_old/docker-build-ascend-a2.yml new file mode 100644 index 00000000000..76540a53c32 --- /dev/null +++ b/.github/workflows_old/docker-build-ascend-a2.yml @@ -0,0 +1,84 @@ +name: docker-build-ascend-a2 + +on: + workflow_dispatch: + push: + branches: ["main"] + paths: + - "docker/ascend/Dockerfile.ascend_8.5.0_a2" + - ".github/workflows/docker-build-ascend-a2.yml" + release: + types: [published] + schedule: + - cron: "0 16 * * *" + +jobs: + build-ascend-image-a2: + if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'verl-project' }} + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-build-ascend-image-a2 + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + steps: + - name: Remove unnecessary parts in github actions runners to free up disk space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Get base image name and tag + id: base_image + run: | + BASE_IMAGE_FULL=$(grep '^FROM' ./docker/ascend/Dockerfile.ascend_8.5.0_a2 | head -1 | cut -d' ' -f2) + echo "Base image full: $BASE_IMAGE_FULL" + BASE_IMAGE_TAG=$(echo "$BASE_IMAGE_FULL" | cut -d':' -f2) + echo "Base image tag: $BASE_IMAGE_TAG" + NEW_IMAGE_NAME="verl-$BASE_IMAGE_TAG" + echo "New image name: $NEW_IMAGE_NAME" + echo "base_image_tag=$BASE_IMAGE_TAG" >> "$GITHUB_OUTPUT" + echo "new_image_name=$NEW_IMAGE_NAME" >> "$GITHUB_OUTPUT" + + - name: Get image tag + id: version + run: | + BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///g' | sed 's/[^a-zA-Z0-9._-]/_/g') + if [ "${{ github.event_name }}" = "release" ]; then + echo "tag=${{ steps.base_image.outputs.new_image_name }}-${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT" + elif [ "$BRANCH_NAME" = "main" ]; then + echo "tag=${{ steps.base_image.outputs.new_image_name }}-latest" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Quay.io + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Clean Docker cache before build + run: | + docker system prune -a -f --volumes || true + + - name: Build and push images Quay + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + file: ./docker/ascend/Dockerfile.ascend_8.5.0_a2 + push: true + tags: | + quay.io/ascend/verl:${{ steps.version.outputs.tag }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + BUILDKIT_INLINE_CACHE=1 diff --git a/.github/workflows_old/docker-build-ascend-a3.yml b/.github/workflows_old/docker-build-ascend-a3.yml new file mode 100644 index 00000000000..6549387fddc --- /dev/null +++ b/.github/workflows_old/docker-build-ascend-a3.yml @@ -0,0 +1,84 @@ +name: docker-build-ascend-a3 + +on: + workflow_dispatch: + push: + branches: ["main"] + paths: + - "docker/ascend/Dockerfile.ascend_8.5.0_a3" + - ".github/workflows/docker-build-ascend-a3.yml" + release: + types: [published] + schedule: + - cron: "0 19 * * *" + +jobs: + build-ascend-image-a3: + if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'verl-project' }} + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-build-ascend-image-a3 + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + steps: + - name: Remove unnecessary parts in github actions runners to free up disk space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Get base image name and tag + id: base_image + run: | + BASE_IMAGE_FULL=$(grep '^FROM' ./docker/ascend/Dockerfile.ascend_8.5.0_a3 | head -1 | cut -d' ' -f2) + echo "Base image full: $BASE_IMAGE_FULL" + BASE_IMAGE_TAG=$(echo "$BASE_IMAGE_FULL" | cut -d':' -f2) + echo "Base image tag: $BASE_IMAGE_TAG" + NEW_IMAGE_NAME="verl-$BASE_IMAGE_TAG" + echo "New image name: $NEW_IMAGE_NAME" + echo "base_image_tag=$BASE_IMAGE_TAG" >> "$GITHUB_OUTPUT" + echo "new_image_name=$NEW_IMAGE_NAME" >> "$GITHUB_OUTPUT" + + - name: Get image tag + id: version + run: | + BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///g' | sed 's/[^a-zA-Z0-9._-]/_/g') + if [ "${{ github.event_name }}" = "release" ]; then + echo "tag=${{ steps.base_image.outputs.new_image_name }}-${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT" + elif [ "$BRANCH_NAME" = "main" ]; then + echo "tag=${{ steps.base_image.outputs.new_image_name }}-latest" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Quay.io + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Clean Docker cache before build + run: | + docker system prune -a -f --volumes || true + + - name: Build and push images Quay + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + file: ./docker/ascend/Dockerfile.ascend_8.5.0_a3 + push: true + tags: | + quay.io/ascend/verl:${{ steps.version.outputs.tag }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + BUILDKIT_INLINE_CACHE=1 diff --git a/.github/workflows_old/e2e_ascend.yml b/.github/workflows_old/e2e_ascend.yml new file mode 100644 index 00000000000..d4ea77ad143 --- /dev/null +++ b/.github/workflows_old/e2e_ascend.yml @@ -0,0 +1,166 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ascend + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + paths: + - ".github/workflows/e2e_ascend.yml" + - "examples/data_preprocess/**" + - "examples/grpo_trainer/**" + - "examples/ppo_trainer/**" + - "examples/sft/**" + - "verl/experimental/one_step_off_policy/**" + - "tests/special_npu/**" + - "tests/special_sanity/check_device_api_usage.py" + - "verl/**" + - "pyproject.toml" + - "requirements-npu.txt" + - "setup.py" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + +jobs: + llm_rl_job: + if: github.repository_owner == 'verl-project' + name: E2E Ascend testing for RL training scenarios of LLM models + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 120 + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout volcengine/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install -e . + - name: Check final pip list + run: | + pip list + - name: Preprocess gsm8k dataset + run: | + python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k + - name: Running gsm8k e2e training tests with PPO on ASCEND NPU (FSDP backend) + run: | + ray stop --force + bash tests/special_npu/run_qwen3_06b_ppo.sh + rm -rf $HOME/ckpts + - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (FSDP backend) + run: | + ray stop --force + bash tests/special_npu/run_qwen2_5_05b_grpo.sh + rm -rf $HOME/ckpts + - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (MindSpeed backend) + run: | + ray stop --force + USE_DIST_CKPT=True bash tests/special_npu/run_qwen2_5_05b_grpo_mindspeed.sh + rm -rf $HOME/dist_ckpt/qwen2_5_05b_grpo_mindspeed + rm -rf $HOME/ckpts + - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (MindSpeed backend, MoE Model) + run: | + ray stop --force + USE_DIST_CKPT=True USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen3moe_minimal.json DUMMY_MODEL_PATH=$HOME/dist_ckpt/qwen3_30b_grpo_mindspeed bash tests/special_npu/run_qwen3_30b_grpo_mindspeed.sh + - name: Running the E2E test with fully_async_policy algorithm (FSDP2) + run: | + ray stop --force + bash tests/special_npu/run_fully_async_policy.sh + + vlm_rl_job: + if: github.repository_owner == 'verl-project' + name: E2E Ascend testing for RL training scenarios of VLM models + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 120 + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout volcengine/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install -e . + - name: Check final pip list + run: | + pip list + - name: Preprocess geo3k dataset + run: | + python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k + - name: Running geo3k e2e training tests with GRPO on ASCEND NPU + run: | + ray stop --force + bash tests/special_npu/run_qwen2_5_vl_3b_npu.sh + rm -rf $HOME/ckpts diff --git a/.github/workflows_old/e2e_fully_async_policy.yml b/.github/workflows_old/e2e_fully_async_policy.yml new file mode 100644 index 00000000000..a46be304814 --- /dev/null +++ b/.github/workflows_old/e2e_fully_async_policy.yml @@ -0,0 +1,170 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_fully_async_policy + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "!**/*.md" + - "!**/*.sh" + # Other entrypoints + - "!examples/*trainer*" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + - "verl/experimental/fully_async_policy" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "!**/*.md" + - "!**/*.sh" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Home + - "verl/experimental/fully_async_policy" + # Entrypoints + - ".github/workflows/e2e_fully_async_policy.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_fully_async_policy.sh" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + # Test FSDP2 strategy + e2e_fully_async_policy_fsdp2: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 10 # Increase timeout for async training + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + ACTOR_STRATEGY: "fsdp2" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install cupy-cuda12x==13.6.0 + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running the E2E test with fully_async_policy algorithm (FSDP2) + run: | + ray stop --force + bash tests/special_e2e/run_fully_async_policy.sh + + # Test Megatron strategy + e2e_fully_async_policy_megatron: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 10 # Increase timeout for async training + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + ACTOR_STRATEGY: "megatron" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install cupy-cuda12x==13.6.0 + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running the E2E test with fully_async_policy algorithm (Megatron) + run: | + ray stop --force + bash tests/special_e2e/run_fully_async_policy.sh + + cleanup: + runs-on: ubuntu-latest + needs: [setup, e2e_fully_async_policy_fsdp2] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/e2e_one_step_off_policy.yml b/.github/workflows_old/e2e_one_step_off_policy.yml new file mode 100644 index 00000000000..de3f8df5c1e --- /dev/null +++ b/.github/workflows_old/e2e_one_step_off_policy.yml @@ -0,0 +1,171 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_one_step_off_policy + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "!**/*.md" + - "!**/*.sh" + # Other entrypoints + - "!examples/*trainer*" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + - "verl/experimental/one_step_off_policy" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "!**/*.md" + - "!**/*.sh" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Home + - "verl/experimental/one_step_off_policy" + # Entrypoints + - ".github/workflows/e2e_one_step_off_policy.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/run_one_step_off_policy.sh" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + # Test FSDP2 strategy + e2e_one_step_off_policy_fsdp2: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 10 # Increase timeout for async training + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + ACTOR_STRATEGY: "fsdp2" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install cupy-cuda12x==13.6.0 + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running the E2E test with one_step_off_policy algorithm (FSDP2) + run: | + ray stop --force + bash tests/special_e2e/run_one_step_off_policy.sh + + # Test Megatron strategy + e2e_one_step_off_policy_megatron: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 10 # Increase timeout for async training + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + ACTOR_STRATEGY: "megatron" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install cupy-cuda12x==13.6.0 + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running the E2E test with one_step_off_policy algorithm (Megatron) + run: | + ray stop --force + bash tests/special_e2e/run_one_step_off_policy.sh + + cleanup: + runs-on: ubuntu-latest + needs: + [setup, e2e_one_step_off_policy_fsdp2, e2e_one_step_off_policy_megatron] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/e2e_one_step_off_policy_ascend.yml b/.github/workflows_old/e2e_one_step_off_policy_ascend.yml new file mode 100644 index 00000000000..77ed29b4e03 --- /dev/null +++ b/.github/workflows_old/e2e_one_step_off_policy_ascend.yml @@ -0,0 +1,169 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_one_step_off_policy_ascend + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "!**/*.md" + - "!**/*.sh" + # Other entrypoints + - "!examples/*trainer*" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + - "verl/experimental/one_step_off_policy" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - "!**/*.md" + - "!**/*.sh" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Home + - "verl/experimental/one_step_off_policy" + # Entrypoints + - ".github/workflows/e2e_one_step_off_policy_ascend.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_npu/run_one_step_off_policy.sh" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + # Test FSDP2 strategy + e2e_one_step_off_policy_fsdp2_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 60 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + ACTOR_STRATEGY: "fsdp2" + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout verl-project/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e . + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Prepare GSM8K dataset + run: | + python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k + - name: Running the E2E test with one_step_off_policy algorithm (FSDP2) + run: | + ray stop --force + bash tests/special_npu/run_one_step_off_policy.sh + + # Test Megatron strategy + e2e_one_step_off_policy_megatron_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 60 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + ACTOR_STRATEGY: "megatron" + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout verl-project/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e . + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Prepare GSM8K dataset + run: | + python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k + - name: Running the E2E test with one_step_off_policy algorithm (Megatron) + run: | + ray stop --force + bash tests/special_npu/run_one_step_off_policy.sh diff --git a/.github/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml b/.github/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml new file mode 100644 index 00000000000..61a19d43419 --- /dev/null +++ b/.github/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml @@ -0,0 +1,287 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_trtllm + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Recipes + - "!recipe/**" + # FSDP + - "!verl/workers/**/*dp_*.py" + # Entrypoints + - "verl/workers/rollout/trtllm_rollout/**" + - "tests/workers/rollout/rollout_trtllm/**" + - ".github/workflows/e2e_ppo_grpo_trainer_trtllm.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "examples/data_preprocess/dapo_multiturn_w_tool.py" + - "examples/data_preprocess/aime2024_multiturn_w_tool.py" + - "examples/grpo_trainer/run_qwen2-7b_math_trtllm.sh" + - "examples/grpo_trainer/run_qwen2-7b_math_megatron_trtllm.sh" + - "examples/grpo_trainer/run_qwen3-30b_dapo_megatron_fp8_trtllm.sh" + # add back when ppo flow is ready + # - "tests/special_e2e/run_ppo_trainer_megatron.sh" + # - "verl/trainer/main_ppo.py" + # - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:trtllm1.3.0rc4" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + trtllm_unit_tests: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 30 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install pytest-asyncio + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Run TRTLLM unit tests + run: | + export TRTLLM_TEST_MODEL_PATH_ROOT="${HOME}/models" + ray stop --force + pytest -v -s \ + tests/workers/rollout/rollout_trtllm/test_adapter.py \ + tests/workers/rollout/rollout_trtllm/test_async_server.py \ + tests/workers/rollout/rollout_trtllm/test_trtllm_rollout_utils.py + + e2e_grpo_trainer_fsdp-qwen2: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 30 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_save_dir ${PWD}/data/gsm8k + - name: Running GSM8K E2E training tests with FSDP on 8 L20 GPUs (Qwen) + run: | + ray stop --force + DATADIR=${HOME}/data \ + bash examples/grpo_trainer/run_qwen2-7b_math_trtllm.sh 2 \ + trainer.total_training_steps=1 \ + data.train_files="['${PWD}/data/gsm8k/train.parquet']" \ + data.val_files="['${PWD}/data/gsm8k/test.parquet']" \ + trainer.logger='["console"]' \ + actor_rollout_ref.model.path="${HOME}/models/Qwen/Qwen2.5-0.5B-Instruct" + - name: clean up + run: | + rm -rf checkpoints + + e2e_grpo_trainer_megatron-qwen2: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 30 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_save_dir ${PWD}/data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen) + run: | + ray stop --force + DATADIR=${HOME}/data \ + ACTOR_TP=2 \ + bash examples/grpo_trainer/run_qwen2-7b_math_megatron_trtllm.sh 2 \ + trainer.total_training_steps=1 \ + data.train_files="['${PWD}/data/gsm8k/train.parquet']" \ + data.val_files="['${PWD}/data/gsm8k/test.parquet']" \ + trainer.logger='["console"]' \ + actor_rollout_ref.model.path="${HOME}/models/Qwen/Qwen2.5-0.5B-Instruct" + - name: clean up + run: | + rm -rf checkpoints + e2e_grpo_trainer_fsdp-vlm: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 30 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install qwen_vl_utils + pip3 install mathruler + - name: Prepare GEO3K dataset + run: | + python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k --local_save_dir ${PWD}/data/geo3k + - name: Running GEO3K E2E training tests with FSDP on 8 L20 GPUs (VLM) + run: | + ray stop --force + DATADIR=${HOME}/data \ + bash examples/grpo_trainer/run_qwen2_5_vl_3b_trtllm.sh 2 \ + trainer.total_training_steps=1 \ + data.train_files="['${PWD}/data/geo3k/train.parquet']" \ + data.val_files="['${PWD}/data/geo3k/test.parquet']" \ + trainer.logger='["console"]' \ + actor_rollout_ref.model.path="${HOME}/models/Qwen/Qwen3-VL-2B-Instruct" + - name: clean up + run: | + rm -rf checkpoints + - name: Prepare DAPO-Math-17k and AIME-2024 datasets (data_preprocess) + run: | + python3 examples/data_preprocess/dapo_multiturn_w_tool.py --local_save_dir ${PWD}/data/dapo-math-17k + python3 examples/data_preprocess/aime2024_multiturn_w_tool.py --local_save_dir ${PWD}/data/aime-2024 + - name: Running DAPO E2E with FP8 TRT-LLM rollout (Qwen3-0.6B) + run: | + ray stop --force + export INFER_TP=2 ACTOR_TP=2 ACTOR_PP=2 ACTOR_VPP=2 ACTOR_EP=1 ACTOR_CP=2 REF_TP=2 REF_PP=2 REF_VPP=2 REF_EP=1 REF_CP=2 GEN_MOE_TP=null GEN_MOE_EP=null + export NNODES=1 GPUS_PER_NODE=8 TRTLLM_MOE_BACKEND=CUTLASS + export DATA_DIR=${PWD} DAPO_MATH_TRAIN=${PWD}/data/dapo-math-17k/train.parquet AIME_VAL=${PWD}/data/aime-2024/train.parquet MODEL_PATH=${HOME}/models/Qwen/Qwen3-0.6B + bash examples/grpo_trainer/run_qwen3-30b_dapo_megatron_fp8_trtllm.sh \ + reward_model.reward_kwargs.overlong_buffer_cfg.len=258 \ + reward_model.reward_kwargs.max_resp_len=512 \ + data.max_prompt_length=512 \ + data.max_response_length=512 \ + data.train_batch_size=32 \ + actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.rollout.max_num_seqs=16 \ + actor_rollout_ref.rollout.max_num_batched_tokens=1024 \ + actor_rollout_ref.rollout.max_model_len=1024 \ + actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=False \ + actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=False \ + trainer.total_training_steps=1 \ + trainer.logger='["console"]' + - name: clean up + run: | + rm -rf checkpoints + + cleanup: + runs-on: ubuntu-latest + needs: [setup, trtllm_unit_tests, e2e_grpo_trainer_fsdp-qwen2, e2e_grpo_trainer_megatron-qwen2, e2e_grpo_trainer_fsdp-vlm] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/e2e_ppo_trainer.yml b/.github/workflows_old/e2e_ppo_trainer.yml new file mode 100644 index 00000000000..357f0aa6bb6 --- /dev/null +++ b/.github/workflows_old/e2e_ppo_trainer.yml @@ -0,0 +1,78 @@ +name: e2e_ppo_trainer + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + + # Megatron + - "!verl/workers/**/megatron_*.py" + + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!**/*.md" + - "!docker/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Docs + - "!docs/**" + + # Megatron + - "!verl/workers/**/megatron_*.py" + # Entrypoints + - ".github/workflows/e2e_ppo_trainer.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/ppo_trainer" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + pre_commit_for_ppo: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install the current repository + run: | + pip install pre-commit hydra-core + pip3 install --no-deps -e . + - name: Set ruff --output-format=github + run: | + sed -i 's/--output-format=full/--output-format=github/' .pre-commit-config.yaml + git add .pre-commit-config.yaml + - uses: pre-commit/action@v3.0.1 + with: + extra_args: "" # Overriding default "--all-files" + diff --git a/.github/workflows_old/e2e_ppo_trainer_megatron_sglang.yml b/.github/workflows_old/e2e_ppo_trainer_megatron_sglang.yml new file mode 100644 index 00000000000..5a8ef80432e --- /dev/null +++ b/.github/workflows_old/e2e_ppo_trainer_megatron_sglang.yml @@ -0,0 +1,201 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_sglang + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" # FSDP + - "!verl/workers/**/*dp_*.py" + - "!verl/utils/fsdp_utils.py" + - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" + - "!verl/model_merger/fsdp_model_merger.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" # FSDP + - "!verl/workers/**/*dp_*.py" + - "!verl/utils/fsdp_utils.py" + - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" + - "!verl/model_merger/fsdp_model_merger.py" + # Entrypoints + - "verl/worksers/rollout/sglang_rollout/*" + - ".github/workflows/e2e_ppo_trainer_megatron_sglang.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_ppo_trainer_megatron-deepseek: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + ENGINE: sglang + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) + run: | + ray stop --force + OPTIM_MEMORY_EFFICIENT=True ENGINE=sglang SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) + run: | + ray stop --force + export VLLM_USE_V1=1 + ray start --head + ENGINE=sglang MODE=async RESUME_MODE=auto MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct TOTAL_TRAIN_STEPS=2 bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Profiling GRPO GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Deepseek) + run: | + ray stop --force + PROFILE_ENABLE=True ENGINE=sglang ADV_ESTIMATOR=grpo USE_DYNAMIC_BSZ=False MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct bash tests/special_e2e/run_ppo_trainer_megatron.sh + if [ -z "$( ls -A '/tmp/ray/session_latest/logs/nsight/' )" ]; then + echo "[ERROR] not found any profiling files" + exit 1 + else + echo "[SUCCESS] profile success" + fi + - name: clean up + run: | + rm -rf checkpoints + + # Qwen3-0.6B: dense, tie_word_embeddings=True + e2e_ppo_trainer_megatron-qwen3: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + ENGINE: sglang + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen3) testing learning rate scheduler + run: | + ray stop --force + ALL_OFFLOAD=True VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 LR_WARMUP_STEPS=1 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with FP8 rollout + run: | + ray stop --force + export VLLM_USE_V1=1 + ROLLOUT_QUANTIZATION=fp8 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + + cleanup: + runs-on: ubuntu-latest + needs: + [setup, e2e_ppo_trainer_megatron-deepseek, e2e_ppo_trainer_megatron-qwen3] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml b/.github/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml new file mode 100644 index 00000000000..ba9d3b23545 --- /dev/null +++ b/.github/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml @@ -0,0 +1,201 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_sglang_2 + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" # FSDP + - "!verl/workers/**/*dp_*.py" + - "!verl/utils/fsdp_utils.py" + - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" + - "!verl/model_merger/fsdp_model_merger.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" # FSDP + - "!verl/workers/**/*dp_*.py" + - "!verl/utils/fsdp_utils.py" + - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" + - "!verl/model_merger/fsdp_model_merger.py" + # Entrypoints + - "verl/worksers/rollout/sglang_rollout/*" + - ".github/workflows/e2e_ppo_trainer_megatron_sglang.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_ppo_trainer_fsdp_sglang: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm and save ckpt + run: | + ray stop --force + ENGINE=sglang bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + e2e_ppo_trainer_fsdp-qwen2_5vl-3b: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + # Geo3k + - name: Prepare GEO3K dataset + run: | + ray stop --force + python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ + - name: Running GEO3K VLM E2E training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + ENGINE=sglang ROLLOUT_MODE=async GPU_MEMORY_UTILIZATION=0.6 ACTOR_FSDP_PARAM_OFFLOAD=True \ + ACTOR_FSDP_OPTIMIZER_OFFLOAD=True REF_FSDP_PARAM_OFFLOAD=True \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GEO3K VLM E2E with rmpad using torch fused kernel (Qwen2.5-VL) + run: | + ray stop --force + FUSED_KERNELS=True TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + ENGINE=sglang ROLLOUT_MODE=async GPU_MEMORY_UTILIZATION=0.6 ACTOR_FSDP_PARAM_OFFLOAD=True \ + ACTOR_FSDP_OPTIMIZER_OFFLOAD=True REF_FSDP_PARAM_OFFLOAD=True \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GEO3K VLM E2E with rmpad using triton fused kernel (Qwen2.5-VL) + run: | + ray stop --force + FUSED_KERNELS=True FUSED_KERNEL_BACKEND=triton \ + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + ENGINE=sglang ROLLOUT_MODE=async GPU_MEMORY_UTILIZATION=0.6 ACTOR_FSDP_PARAM_OFFLOAD=True \ + ACTOR_FSDP_OPTIMIZER_OFFLOAD=True REF_FSDP_PARAM_OFFLOAD=True \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + cleanup: + runs-on: ubuntu-latest + needs: + [setup, e2e_ppo_trainer_fsdp-qwen2_5vl-3b, e2e_ppo_trainer_fsdp_sglang] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/e2e_ppo_trainer_megatron_vllm.yml b/.github/workflows_old/e2e_ppo_trainer_megatron_vllm.yml new file mode 100644 index 00000000000..d9fa832a56b --- /dev/null +++ b/.github/workflows_old/e2e_ppo_trainer_megatron_vllm.yml @@ -0,0 +1,212 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_vllm + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + # FSDP + - "!verl/workers/**/*dp_*.py" + - "!verl/utils/fsdp_utils.py" + - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" + - "!verl/model_merger/fsdp_model_merger.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # FSDP + - "!verl/workers/**/*dp_*.py" + - "!verl/utils/fsdp_utils.py" + - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" + - "!verl/model_merger/fsdp_model_merger.py" + # Entrypoints + - ".github/workflows/e2e_ppo_trainer_megatron_vllm.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + # deepseek-ai/deepseek-coder-1.3b-instruct: dense, tie_word_embeddings=False + e2e_ppo_trainer_megatron-deepseek: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps --force-reinstall . + pip3 install mbridge + pip3 install math-verify + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + # Full training save&load + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron, use mbridge e2e to pre-load and save (Deepseek) + run: | + ray stop --force + ALL_OFFLOAD=True SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 USE_MBRIDGE=True USE_DIST_CKPT=False \ + bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron, use mbridge e2e to pre-load and save (Deepseek) + run: | + ray stop --force + RESUME_MODE=auto MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct TOTAL_TRAIN_STEPS=2 SAVE_FREQ=1 COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 USE_MBRIDGE=True USE_DIST_CKPT=False \ + bash tests/special_e2e/run_ppo_trainer_megatron.sh + # LoRA training save&load + - name: clean up and install Megatron-Bridge + run: | + rm -rf checkpoints + pip3 install git+https://github.com/NVIDIA-NeMo/Megatron-Bridge.git@83a7c11 --no-deps --no-build-isolation + pip3 install git+https://github.com/NVIDIA/Megatron-LM.git@5455f0a --no-deps --no-build-isolation + pip3 install "nvidia-modelopt[torch]>=0.37.0" transformers==4.57.1 + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron, use Megatron-Bridge LoRA e2e to pre-load and save (Deepseek) + run: | + ray stop --force + ALL_OFFLOAD=True SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct COMMON_PP=4 LORA_RANK=8 COMMON_VPP=null COMMON_CP=1 USE_MBRIDGE=True VANILLA_MBRIDGE=False VALUE_VANILLA_MBRIDGE=False USE_DIST_CKPT=False \ + bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron, use Megatron-Bridge LoRA e2e to pre-load and save (Deepseek) + run: | + ray stop --force + RESUME_MODE=auto MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct TOTAL_TRAIN_STEPS=2 SAVE_FREQ=1 COMMON_PP=4 LORA_RANK=8 COMMON_VPP=null COMMON_CP=1 USE_MBRIDGE=True VANILLA_MBRIDGE=False VALUE_VANILLA_MBRIDGE=False USE_DIST_CKPT=False \ + bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + + # Qwen3-0.6B: dense, tie_word_embeddings=True + e2e_ppo_trainer_megatron-qwen3: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install math-verify + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen3) testing learning rate scheduler + run: | + ray stop --force + ALL_OFFLOAD=True VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 LR_WARMUP_STEPS=1 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with FP8 rollout + run: | + ray stop --force + export VLLM_USE_V1=1 + ROLLOUT_QUANTIZATION=fp8 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + + cleanup: + runs-on: ubuntu-latest + needs: + [setup, e2e_ppo_trainer_megatron-deepseek, e2e_ppo_trainer_megatron-qwen3] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml b/.github/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml new file mode 100644 index 00000000000..83a1faf8832 --- /dev/null +++ b/.github/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml @@ -0,0 +1,318 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_vllm_2 + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + # FSDP + - "!verl/workers/**/*dp_*.py" + - "!verl/utils/fsdp_utils.py" + - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" + - "!verl/model_merger/fsdp_model_merger.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # FSDP + - "!verl/workers/**/*dp_*.py" + - "!verl/utils/fsdp_utils.py" + - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" + - "!verl/model_merger/fsdp_model_merger.py" + # Entrypoints + - ".github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_ppo_trainer_megatron-moe-expert-parallel: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps --force-reinstall . + pip3 install git+https://github.com/NVIDIA-NeMo/Megatron-Bridge.git@83a7c11 --no-deps --no-build-isolation + pip3 install git+https://github.com/NVIDIA/Megatron-LM.git@5455f0a --no-deps --no-build-isolation + pip3 install "nvidia-modelopt[torch]>=0.37.0" transformers==4.57.1 + - name: Prepare GSM8K dataset + run: | + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron-Bridge (Qwen3-30B-A3B-Instruct-2507) + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json \ + PPO_MAX_TOKEN_LEN=1024 FWD_MAX_TOKEN_LEN=1024 \ + MAX_PROMPT_LENGTH=512 MAX_RESPONSE_LENGTH=512 \ + MODEL_ID=Qwen/Qwen3-30B-A3B-Instruct-2507 USE_MBRIDGE=True VANILLA_MBRIDGE=False VALUE_VANILLA_MBRIDGE=False \ + COMMON_PP=2 COMMON_VPP=null COMMON_CP=1 COMMON_TP=4 COMMON_EP=4 COMMON_ETP=1 INFER_TP=8 \ + USE_DIST_CKPT=True ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: Running GSM8K E2E training tests with 3D parallelism with FP8 rollout on 8 L20 GPUs with Megatron-Bridge (Qwen3-30B-A3B-Instruct-2507) + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json \ + PPO_MAX_TOKEN_LEN=1024 FWD_MAX_TOKEN_LEN=1024 \ + MAX_PROMPT_LENGTH=512 MAX_RESPONSE_LENGTH=512 \ + MODEL_ID=Qwen/Qwen3-30B-A3B-Instruct-2507 USE_MBRIDGE=True VANILLA_MBRIDGE=False VALUE_VANILLA_MBRIDGE=False \ + COMMON_PP=2 COMMON_VPP=null COMMON_CP=1 COMMON_TP=4 COMMON_EP=4 COMMON_ETP=1 INFER_TP=2 \ + USE_DIST_CKPT=True ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 ROLLOUT_QUANTIZATION=fp8 bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron-Bridge LoRA (Qwen3-30B-A3B-Instruct-2507) + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json \ + PPO_MAX_TOKEN_LEN=1024 FWD_MAX_TOKEN_LEN=1024 \ + MAX_PROMPT_LENGTH=512 MAX_RESPONSE_LENGTH=512 LORA_RANK=8 CRITIC_LORA_RANK=8 \ + MODEL_ID=Qwen/Qwen3-30B-A3B-Instruct-2507 USE_MBRIDGE=True VANILLA_MBRIDGE=False VALUE_VANILLA_MBRIDGE=False \ + COMMON_PP=2 COMMON_VPP=null COMMON_CP=1 COMMON_TP=4 COMMON_EP=2 COMMON_ETP=1 INFER_TP=8 \ + USE_DIST_CKPT=False LORA_MERGE=True ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 bash tests/special_e2e/run_ppo_trainer_megatron.sh + - name: clean up + run: | + rm -rf checkpoints + + e2e_ppo_trainer_fsdp_vllm: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Prepare GSM8K dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + # Function RM + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (FSDP_SIZE=8) + run: | + ray stop --force + VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp-size8" bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm after resuming + run: | + ray stop --force + RESUME_MODE=auto VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp-size8" bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test merging FSDP checkpoints (Qwen Actor) + run: | + exp_name="qwen2.5-0.5b-function-reward-minimal-fsdp-size8" + python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (DDP_SIZE=2, FSDP_SIZE=4) + run: | + ray stop --force + VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True FSDP_SIZE=4 USE_KL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-ddp-size2-fsdp-size4" bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test merging DDP+FSDP checkpoints (Qwen Actor) + run: | + exp_name="qwen2.5-0.5b-function-reward-minimal-ddp-size2-fsdp-size4" + python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (FSDP2) + run: | + ray stop --force + VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp2-size8" STRATEGY=fsdp2 bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test merging FSDP2 checkpoints (Qwen Actor) + run: | + exp_name="qwen2.5-0.5b-function-reward-minimal-fsdp2-size8" + python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + - name: Running GSM8K E2E without rmpad using function rm + run: | + ray stop --force + RM_PAD=False bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm (GRPO) + run: | + ray stop --force + CUSTOM_REWARD_FN=True ADV_ESTIMATOR=grpo USE_KL=True bash tests/special_e2e/ppo_trainer/run_function_reward.sh + # - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm (ReMax) + # run: | + # ray stop --force + # ADV_ESTIMATOR=remax USE_KL=True bash tests/special_e2e/ppo_trainer/run_function_reward.sh + # LoRA tests + - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm and layered_summon + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors LAYERED_SUMMON=True TOTAL_TRAIN_STEPS=1 SAVE_FREQ=1 FSDP_SIZE=4 VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal" bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test GRPO LoRA checkpoints merging function + run: | + export EXP_NAME="qwen2.5-0.5b-function-reward-minimal" + ls checkpoints/verl-test/${EXP_NAME}/global_step_1/actor + cat checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/huggingface/config.json + python3 -m verl.model_merger merge --backend fsdp --local_dir checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/ --target_dir checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/huggingface + - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm and layered_summon with fsdp2 + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors LAYERED_SUMMON=True STRATEGY=fsdp2 bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + e2e_ppo_trainer_fsdp-qwen2_5vl-3b: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 40 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + # Geo3k + - name: Prepare GEO3K dataset + run: | + python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ + - name: Running GEO3K VLM GRPO E2E training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + SP_SIZE=2 \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + - name: Running GEO3K VLM PPO E2E training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=gae RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + SP_SIZE=2 \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GEO3K VLM GRPO E2E lora training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + SP_SIZE=2 \ + LORA_RANK=32 LORA_EXCLUDE=".*visual.*" \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_ppo_trainer_megatron-moe-expert-parallel, + e2e_ppo_trainer_fsdp-qwen2_5vl-3b, + e2e_ppo_trainer_fsdp_vllm, + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml b/.github/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml new file mode 100644 index 00000000000..d0abdcc60e3 --- /dev/null +++ b/.github/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml @@ -0,0 +1,233 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_megatron_vllm_2_ascend + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + # FSDP + - "!verl/workers/**/*dp_*.py" + - "!verl/utils/fsdp_utils.py" + - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" + - "!verl/model_merger/fsdp_model_merger.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # FSDP + - "!verl/workers/**/*dp_*.py" + - "!verl/utils/fsdp_utils.py" + - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" + - "!verl/model_merger/fsdp_model_merger.py" + # Entrypoints + - ".github/workflows/e2e_ppo_trainer_megatron_vllm_2_ascend.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_megatron.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_megatron_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + e2e_ppo_trainer_fsdp_vllm_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 90 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout verl-project/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e . + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Prepare GSM8K dataset + run: | + python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k + # Function RM + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (DDP_SIZE=2, FSDP_SIZE=4) + run: | + ray stop --force + VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True FSDP_SIZE=4 USE_KL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-ddp-size2-fsdp-size4" bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test merging DDP+FSDP checkpoints (Qwen Actor) + run: | + exp_name="qwen2.5-0.5b-function-reward-minimal-ddp-size2-fsdp-size4" + python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (FSDP2) + run: | + ray stop --force + VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp2-size8" STRATEGY=fsdp2 bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test merging FSDP2 checkpoints (Qwen Actor) + run: | + exp_name="qwen2.5-0.5b-function-reward-minimal-fsdp2-size8" + python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface + - name: Running GSM8K E2E without rmpad using function rm + run: | + ray stop --force + RM_PAD=False bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm (GRPO) + run: | + ray stop --force + CUSTOM_REWARD_FN=True ADV_ESTIMATOR=grpo USE_KL=True bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm and layered_summon + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors LAYERED_SUMMON=True TOTAL_TRAIN_STEPS=1 SAVE_FREQ=1 FSDP_SIZE=4 VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal" bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Test GRPO LoRA checkpoints merging function + run: | + export EXP_NAME="qwen2.5-0.5b-function-reward-minimal" + ls checkpoints/verl-test/${EXP_NAME}/global_step_1/actor + cat checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/huggingface/config.json + python3 -m verl.model_merger merge --backend fsdp --local_dir checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/ --target_dir checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/huggingface + - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm and layered_summon with fsdp2 + run: | + ray stop --force + ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors LAYERED_SUMMON=True STRATEGY=fsdp2 bash tests/special_e2e/ppo_trainer/run_function_reward.sh + + e2e_ppo_trainer_fsdp-qwen2_5vl-3b_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 60 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout verl-project/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e . + pip install trl==0.26.0 + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + # Geo3k + - name: Prepare GEO3K dataset + run: | + python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k + - name: Running GEO3K VLM GRPO E2E training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + SP_SIZE=2 \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GEO3K VLM PPO E2E training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=gae RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + SP_SIZE=2 \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh + - name: Running GEO3K VLM GRPO E2E lora training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ + MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ + MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ + ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ + SP_SIZE=2 \ + LORA_RANK=32 LORA_EXCLUDE=".*visual.*" \ + bash tests/special_e2e/ppo_trainer/run_function_reward.sh diff --git a/.github/workflows_old/e2e_ppo_trainer_veomni_vllm.yml b/.github/workflows_old/e2e_ppo_trainer_veomni_vllm.yml new file mode 100644 index 00000000000..0accafd58e8 --- /dev/null +++ b/.github/workflows_old/e2e_ppo_trainer_veomni_vllm.yml @@ -0,0 +1,153 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_ppo_trainer_veomni_vllm + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch. + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!verl/trainer/fsdp_sft_trainer.py" + # Megatron + - "!verl/workers/**/megatron_*.py" + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!docker/**" + # Docs + - "!**/*.md" + - "!docs/**" + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Megatron + - "!verl/workers/**/megatron_*.py" + # Entrypoints + - ".github/workflows/e2e_ppo_trainer_veomni_vllm.yml" + - "examples/data_preprocess/gsm8k.py" + - "examples/data_preprocess/geo3k.py" + - "tests/special_e2e/run_ppo_trainer_veomni.sh" + - "verl/trainer/main_ppo.py" + - "verl/trainer/config/ppo_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + e2e_ppo_trainer_veomni_vllm: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install git+https://github.com/ByteDance-Seed/VeOmni.git@v0.1.4 + - name: Prepare GSM8K dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Prepare GEO3K dataset + run: | + ray stop --force + python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ + - name: Running GSM8K E2E training tests on 8 L20 GPUs with veomni engine (FSDP_SIZE=4, USP=2) + run: | + ray stop --force + FSDP_SIZE=4 SP_SIZE=2 bash tests/special_e2e/run_ppo_trainer_veomni.sh + - name: Running GEO3K E2E training tests on 8 L20 GPUs with veomni engine (FSDP_SIZE=8, USP=1) + run: | + ray stop --force + MODEL_ID=Qwen/Qwen3-VL-2B-Instruct TRAIN_FILES=${HOME}/data/geo3k/train.parquet VAL_FILES=${HOME}/data/gsm8k/test.parquet FSDP_SIZE=8 SP_SIZE=1 bash tests/special_e2e/run_ppo_trainer_veomni.sh + + cleanup: + runs-on: ubuntu-latest + needs: + [ + setup, + e2e_ppo_trainer_veomni_vllm, + ] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/e2e_sft_llm.yml b/.github/workflows_old/e2e_sft_llm.yml new file mode 100644 index 00000000000..435a0a626db --- /dev/null +++ b/.github/workflows_old/e2e_sft_llm.yml @@ -0,0 +1,153 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_sft_llm + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + + # Megatron + - "!verl/workers/**/megatron_*.py" + # Entrypoints + - ".github/workflows/e2e_sft_llm.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/sft" + - "verl/trainer/fsdp_sft_trainer.py" + - "verl/trainer/config/sft_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + e2e_sft_llm: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 30 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install peft + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install git+https://github.com/ByteDance-Seed/VeOmni.git@v0.1.4 + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k_multiturn_sft.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm + run: | + ray stop --force + bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs w/o rmpad using function rm + run: | + ray stop --force + RM_PAD=False bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with sequence parallism + run: | + ray stop --force + SP_SIZE=2 bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests on 8 L20 GPUs with sequence parallism and liger + run: | + ray stop --force + SP_SIZE=2 LIGER=True bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests with LoRA + run: | + ray stop --force + LORA_RANK=32 bash tests/special_e2e/sft/run_sft.sh + - name: Run GSM8K E2E training and resume tests resuming from the checkpoint manager + run: | + ray stop --force + LORA_RANK=32 RESUME_MODE=auto TOTAL_TRAIN_STEP=2 bash tests/special_e2e/sft/run_sft.sh + # TODO: multiturn + - name: Running GSM8K E2E training tests with multiturn and various configs and compare results + run: | + bash tests/special_e2e/sft/test_sft_engine_all.sh + + cleanup: + runs-on: ubuntu-latest + needs: [setup, e2e_sft_llm] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/e2e_sft_llm_ascend.yml b/.github/workflows_old/e2e_sft_llm_ascend.yml new file mode 100644 index 00000000000..3919da747a9 --- /dev/null +++ b/.github/workflows_old/e2e_sft_llm_ascend.yml @@ -0,0 +1,160 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_sft_llm_ascend + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + + # Megatron + - "!verl/workers/**/megatron_*.py" + # Entrypoints + - ".github/workflows/e2e_sft_llm_ascend.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/sft" + - "verl/trainer/fsdp_sft_trainer.py" + - "verl/trainer/config/sft_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + e2e_sft_llm_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 90 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout verl-project/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install -e . + pip install git+https://github.com/ByteDance-Seed/VeOmni.git@v0.1.4 + pip install pandas==2.3.3 + pip uninstall -y mbridge + pip install git+https://github.com/ISEEKYAN/mbridge.git@89eb10 + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Prepare gsm8k dataset + run: | + python3 examples/data_preprocess/gsm8k_multiturn_sft.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k + - name: Running GSM8K E2E training tests on 8 NPUs with rmpad using function rm + run: | + ray stop --force + bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests on 8 NPUs w/o rmpad using function rm + run: | + ray stop --force + RM_PAD=False bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests on 8 NPUs with sequence parallism + run: | + ray stop --force + SP_SIZE=2 bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests with LoRA + run: | + ray stop --force + LORA_RANK=32 bash tests/special_e2e/sft/run_sft.sh + - name: Run GSM8K E2E training and resume tests resuming from the checkpoint manager + run: | + ray stop --force + LORA_RANK=32 RESUME_MODE=auto TOTAL_TRAIN_STEP=2 bash tests/special_e2e/sft/run_sft.sh + - name: Running GSM8K E2E training tests with multiturn and various configs and compare results + run: | + ray stop --force + rm -rf ~/verl/test/log + mkdir -p ~/verl/test/log + export VERL_FILE_LOGGER_ROOT=~/verl/test/log + # test with single gpu as golden + echo "run with single gpu as golden" + BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=1 NUM_GPUS=1 FSDP_STRATEGY=fsdp VERL_FILE_LOGGER_PATH=~/verl/test/log/golden.jsonl bash tests/special_e2e/sft/run_sft_engine.sh + # test with fsdp 1 + echo "run with sp2 fsdp_size2 num_gpus8 fsdp_strategy fsdp pad_mode no_padding" + BACKEND=fsdp SP_SIZE=2 FSDP_SIZE=2 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=no_padding bash tests/special_e2e/sft/run_sft_engine.sh + # test with fsdp 1 use_remove_padding and pad_mode no_padding + echo "run with sp4 fsdp_size4 num_gpus8 fsdp_strategy fsdp pad_mode no_padding use_remove_padding False" + BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=-1 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=no_padding USE_REMOVE_PADDING=False bash tests/special_e2e/sft/run_sft_engine.sh + # test with fsdp 2 + echo "run with sp2 fsdp_size2 num_gpus8 fsdp_strategy fsdp2" + BACKEND=fsdp SP_SIZE=2 FSDP_SIZE=2 NUM_GPUS=8 FSDP_STRATEGY=fsdp2 bash tests/special_e2e/sft/run_sft_engine.sh + # test with veomni + echo "run with sp2 fsdp_size4 num_gpus8 fsdp_strategy fsdp2" + BACKEND=veomni SP_SIZE=2 FSDP_SIZE=4 NUM_GPUS=8 FSDP_STRATEGY=fsdp2 bash tests/special_e2e/sft/run_sft_engine.sh + # test with megatron + echo "run with tp2 pp2 vpp2 cp2 num_gpus8" + BACKEND=megatron TP_SIZE=2 PP_SIZE=2 VPP_SIZE=NULL CP_SIZE=2 NUM_GPUS=8 bash tests/special_e2e/sft/run_sft_engine.sh + # test with cp in ray + echo "run with tp2 pp2 vpp2 cp2 num_gpus8 mode=ray" + BACKEND=megatron TP_SIZE=2 PP_SIZE=2 VPP_SIZE=NULL CP_SIZE=2 NUM_GPUS=8 mode=ray bash tests/special_e2e/sft/run_sft_engine.sh + rm -rf ~/verl/test/log diff --git a/.github/workflows_old/e2e_sft_vlm.yml b/.github/workflows_old/e2e_sft_vlm.yml new file mode 100644 index 00000000000..93d02c83c8c --- /dev/null +++ b/.github/workflows_old/e2e_sft_vlm.yml @@ -0,0 +1,128 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: e2e_sft_vlm + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + + # Megatron + - "!verl/workers/**/megatron_*.py" + # Entrypoints + - ".github/workflows/e2e_sft_vlm.yml" + - "examples/data_preprocess/gsm8k.py" + - "tests/special_e2e/sft" + - "verl/trainer/fsdp_sft_trainer.py" + - "verl/trainer/config/sft_trainer.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + e2e_sft_vlm: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 30 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install peft + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install git+https://github.com/ByteDance-Seed/VeOmni.git@v0.1.4 + - name: Prepare pokemon-gpt4o-captions dataset + run: | + ray stop --force + python3 examples/data_preprocess/pokemon.py --local_dataset_path ${HOME}/models/hf_data/pokemon-gpt4o-captions + - name: Running Pokemon E2E training tests with multiturn and various configs and compare results + run: | + MODEL_ID=Qwen/Qwen3-VL-2B-Instruct DATASET_DIR=~/data/pokemon-gpt4o-captions VPP_SIZE=null bash tests/special_e2e/sft/test_sft_engine_all.sh + + cleanup: + runs-on: ubuntu-latest + needs: [setup, e2e_sft_vlm] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/gpu_unit_tests.yml b/.github/workflows_old/gpu_unit_tests.yml new file mode 100644 index 00000000000..6c16b950afd --- /dev/null +++ b/.github/workflows_old/gpu_unit_tests.yml @@ -0,0 +1,137 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: GPU unit tests + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.4.x + paths: + - "**/*.py" + - .github/workflows/gpu_unit_tests.yml + pull_request: + branches: + - main + - v0.4.x + paths: + # The order that you define paths patterns matters: + # A matching negative pattern (prefixed with !) after a positive match will exclude the path. + # A matching positive pattern after a negative match will include the path again. + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # Entrypoints + - .github/workflows/gpu_unit_tests.yml + - "tests/**test_*.py" + # Ignore CPU tests + - "!tests/*_on_cpu.py" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + gpu_unit_tests: + if: github.repository_owner == 'verl-project' + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 60 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1" + HF_HUB_ENABLE_HF_TRANSFER: 1 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install hf_transfer + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install cupy-cuda12x==13.6.0 pytest-asyncio + pip3 install --ignore-installed blinker + pip3 install --ignore-installed mlflow "numpy<2.0" + - name: Run all GPU unit tests + run: | + pytest -s -x --ignore-glob="*on_npu.py" --ignore-glob="*test_special_*.py" --ignore-glob='*on_cpu.py' --ignore-glob="*test_vllm*" --ignore-glob="*_sglang*" --ignore-glob="*_hf_rollout*" --ignore-glob="tests/models/" --ignore-glob='tests/special*' --ignore-glob="tests/experimental" --ignore-glob="tests/workers/reward_model" --ignore-glob="*test_shared_memory*" --ignore-glob="tests/workers/rollout/rollout_trtllm" --ignore-glob="*test_bucketed_weight_transfer*" tests/ + - name: Testing LinearCrossEntropyTP Correctness, Computation Time and Memory Consumption + run: | + LOW_MEMORY=True torchrun --standalone --nnodes=1 --nproc-per-node=8 tests/utils/test_special_linear_cross_entropy_tp.py + - name: Testing FSDP2 actor functionality + run: | + torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/workers/actor/test_special_dp_actor.py + - name: Testing FSDP2 critic functionality + run: | + torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/workers/critic/test_special_dp_critic.py + + cleanup: + runs-on: ubuntu-latest + needs: [setup, gpu_unit_tests] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/model.yml b/.github/workflows_old/model.yml new file mode 100644 index 00000000000..5522ba71466 --- /dev/null +++ b/.github/workflows_old/model.yml @@ -0,0 +1,184 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. +# name: Check PR Title + +name: model + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "verl/**/*.py" + # Entrypoints + - ".github/workflows/model.yml" + - "tests/special_distributed/test_fsdp_ckpt.py" + - "tests/special_distributed/test_tensor_dict.py" + - "tests/models/**" + - "tests/special_distributed/run_all.sh" + +# Declare permissions just read content. +permissions: + contents: read + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + model_rmpad: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository and upgrade to latest transformers(4.54.0)/flash_attn, transformers 4.55.0 has strange behavior with model backward + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install --upgrade "transformers<5.0.0" + - name: Running rmpad model tests on 8 L20 GPUs + flash_attn 2.5.8 + run: | + pytest -s tests/models/test_transformer.py + - name: Running rmpad model tests on 8 L20 GPUs + latest flash_attn + run: | + pytest -s tests/models/test_transformer.py + - name: Running FSDP rmpad model tests on 8 L20 GPUs + latest flash_attn + run: | + STRATEGY=fsdp torchrun --nproc_per_node=8 tests/special_distributed/test_fsdp_ckpt.py + - name: Running transformers ulysses tests on 8 L20 GPUs + latest transformers + run: | + torchrun --nproc_per_node=8 -m pytest tests/models/test_transformers_ulysses.py + - name: Running transformers ulysses tests on 8 L20 GPUs + transformers 4.54.1 + run: | + pip3 install transformers==4.54.1 + torchrun --nproc_per_node=8 -m pytest tests/models/test_transformers_ulysses.py + - name: Run distributed test + run: | + bash tests/special_distributed/run_all.sh + + # TODO: Move this back to model_rmpad once FSDP2 is stable. + # NOTE: List as an independent job to make rerun easier. + model_rmpad_fsdp2_unstable: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository and upgrade to latest transformers/flash_attn + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Running FSDP2 rmpad model tests on 8 L20 GPUs + latest flash_attn + run: | + STRATEGY=fsdp2 torchrun --nproc_per_node=8 tests/special_distributed/test_fsdp_ckpt.py + + model_engine: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 20 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Download model config files + run: | + hf download Qwen/Qwen2.5-0.5B-Instruct --local-dir $HOME/models/Qwen/Qwen2.5-0.5B-Instruct + + - name: Running mcore engine tests on 8 L20 GPUs + run: | + ray stop --force + pytest -s -x tests/models/test_engine.py + + cleanup: + runs-on: ubuntu-latest + needs: [setup, model_rmpad, model_rmpad_fsdp2_unstable, model_engine] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/model_ascend.yml b/.github/workflows_old/model_ascend.yml new file mode 100644 index 00000000000..a5ab7620ee3 --- /dev/null +++ b/.github/workflows_old/model_ascend.yml @@ -0,0 +1,137 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. +# name: Check PR Title + +name: model_ascend + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "verl/**/*.py" + # Entrypoints + - ".github/workflows/model_ascend.yml" + - "tests/special_distributed/test_fsdp_ckpt.py" + - "tests/special_distributed/test_tensor_dict.py" + - "tests/models/**" + - "tests/special_distributed/run_all.sh" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + +jobs: + model_rmpad_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 60 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout verl-project/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e .[test] + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Running rmpad model tests on 8 NPUs + run: | + pytest -s tests/models/test_transformer.py + - name: Running FSDP rmpad model tests on 8 NPUs + run: | + STRATEGY=fsdp torchrun --nproc_per_node=8 tests/special_distributed/test_fsdp_ckpt.py + - name: Running transformers ulysses tests on 8 NPUs + run: | + torchrun --nproc_per_node=8 -m pytest tests/models/test_transformers_ulysses.py + - name: Run distributed test + run: | + bash tests/special_distributed/run_all.sh + + # TODO: Move this back to model_rmpad once FSDP2 is stable. + # NOTE: List as an independent job to make rerun easier. + model_rmpad_fsdp2_unstable_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 60 + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e .[test] + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Running FSDP2 rmpad model tests on 8 NPUs + run: | + STRATEGY=fsdp2 torchrun --nproc_per_node=8 tests/special_distributed/test_fsdp_ckpt.py diff --git a/.github/workflows_old/nightly_ascend.yml b/.github/workflows_old/nightly_ascend.yml new file mode 100644 index 00000000000..c74ea4ba7d9 --- /dev/null +++ b/.github/workflows_old/nightly_ascend.yml @@ -0,0 +1,174 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: nightly_ci_ascend + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + # For push, for now only anti-patterns are specified so it is more conservative + # and achieves higher coverage. + schedule: + - cron: "0 17 * * *" + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + # Test ppo qwen3-8b fsdp+vllm + nightlyCI_ppo-qwen3-8b-fsdp-vllm_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 180 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout verl-project/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e . + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Prepare GSM8K dataset + run: | + python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k + - name: Running nightlyCI_ppo-qwen3-8b-fsdp-vllm_ascend + run: | + ray stop --force + bash tests/special_npu/nightly_ci_ascend/run_ppo_qwen3-8b_fsdp_npu.sh + + # Test grpo qwen25-7b-Instruct fsdp+vllm + nightlyCI_grpo-qwen25-7b-Instruct-fsdp-vllm_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 180 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout verl-project/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e . + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Prepare GSM8K dataset + run: | + python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k + - name: Running nightlyCI_grpo-qwen25-7b-Instruct-fsdp-vllm_ascend + run: | + ray stop --force + bash tests/special_npu/nightly_ci_ascend/run_grpo_qwen25-7b-instruct_fsdp_npu.sh + + # Test grpo qwen25-vl-3b-Instruct fsdp+vllm + nightlyCI_grpo-qwen25-vl-3b-Instruct-fsdp-vllm_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 180 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout verl-project/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e . + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Preprocess geo3k dataset + run: | + python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k + - name: Running nightlyCI_grpo-qwen25-vl-3b-Instruct-fsdp-vllm_ascend + run: | + ray stop --force + bash tests/special_npu/nightly_ci_ascend/run_grpo_qwen25-vl-3b-instruct_fsdp_npu.sh diff --git a/.github/workflows_old/npu_unit_tests.yml b/.github/workflows_old/npu_unit_tests.yml new file mode 100644 index 00000000000..7f678409da0 --- /dev/null +++ b/.github/workflows_old/npu_unit_tests.yml @@ -0,0 +1,126 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - `npu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix on ascend device. +# - Since cpu/gpu/npu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: NPU unit tests + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + - .github/workflows/npu_unit_tests.yml + pull_request: + branches: + - main + paths: + # The order that you define paths patterns matters: + # A matching negative pattern (prefixed with !) after a positive match will exclude the path. + # A matching positive pattern after a negative match will include the path again. + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + - "!recipe/**" + # Entrypoints + - .github/workflows/npu_unit_tests.yml + - "tests/**test_*.py" + # Ignore CPU tests + - "!tests/*_on_cpu.py" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + npu_unit_tests: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 60 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout volcengine/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e .[test] + pip install mlflow pytest-asyncio + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Run all NPU unit tests + run: | + pytest -s -x --ignore-glob="*test_special_*.py" --ignore-glob="*on_cpu.py" --ignore-glob="*test_vllm*" --ignore-glob="*_sglang*" --ignore-glob="*_hf_rollout*" --ignore-glob="tests/models/" --ignore-glob="tests/special*" --ignore-glob="tests/experimental" --ignore-glob="tests/workers/reward_model" --ignore-glob="*test_rvdz*" --ignore-glob="*test_ray_collectives*" --ignore-glob="*test_nvtx_profile*" --ignore-glob="tests/checkpoint_engine" --ignore-glob="*test_shared_memory*" --ignore-glob="tests/workers/rollout/rollout_trtllm" --ignore-glob="*test_fsdp_lora_merge*" --ignore-glob="*test_activation_offload*" --ignore-glob="*test_normalize_peft_param_name.py*" tests/ + - name: Testing activation offload + run: | + pytest -s -x tests/utils/test_activation_offload.py + - name: Testing normalize peft param name + run: | + pytest -s -x tests/utils/test_normalize_peft_param_name.py + - name: Testing FSDP2 actor functionality + run: | + torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/workers/actor/test_special_dp_actor.py + - name: Testing FSDP2 critic functionality + run: | + torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/workers/critic/test_special_dp_critic.py + - name: Running NPU profiling unit tests + run: | + pytest -s -x tests/utils/test_special_mstx_profile.py diff --git a/.github/workflows_old/pre-commit.yml b/.github/workflows_old/pre-commit.yml new file mode 100644 index 00000000000..4f6aa4bdf0d --- /dev/null +++ b/.github/workflows_old/pre-commit.yml @@ -0,0 +1,41 @@ +# c.f. https://github.com/pre-commit/action?tab=readme-ov-file#using-this-action +name: pre-commit + +# No need to avoid / cancel lightweight pre-commit jobs +on: + schedule: + - cron: "0 0 * * 0" + pull_request: + push: + branches: + - main + - v0.* + # Allow manual triggering + workflow_dispatch: + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + pre-commit: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install the current repository + run: | + pip install pre-commit hydra-core + pip install --no-deps -e . + - name: Set ruff --output-format=github + run: | + sed -i 's/--output-format=full/--output-format=github/' .pre-commit-config.yaml + git add .pre-commit-config.yaml + # Check "--all-files" by default + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows_old/precommit-autofix.yml b/.github/workflows_old/precommit-autofix.yml new file mode 100644 index 00000000000..d235da90cd2 --- /dev/null +++ b/.github/workflows_old/precommit-autofix.yml @@ -0,0 +1,52 @@ +name: scheduled pre-commit autofix + +on: + schedule: + # Every hour + - cron: "0 * * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + precommit: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install pre-commit + run: | + python -m pip install --upgrade pip + pip install pre-commit hydra-core + + - name: Run pre-commit + run: | + pre-commit run --all-files || true + + - name: Create or update PR + uses: peter-evans/create-pull-request@v6 + with: + branch: bot/precommit-autofix + delete-branch: true + title: "[ci] chore: scheduled pre-commit autofix" + commit-message: "chore: auto-fix pre-commit issues" + body: | + This PR was created automatically by a scheduled GitHub Action. + + - Runs `pre-commit run --all-files` + - Triggered hourly + labels: | + automated + pre-commit diff --git a/.github/workflows_old/reward_model_sglang.yml b/.github/workflows_old/reward_model_sglang.yml new file mode 100644 index 00000000000..c9a4e9804a0 --- /dev/null +++ b/.github/workflows_old/reward_model_sglang.yml @@ -0,0 +1,134 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. +# name: Check PR Title + +name: reward_model_sglang + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "verl/**/*.py" + # Entrypoints + - ".github/workflows/reward_model_sglang.yml" + - "tests/experimental/reward_loop/**" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + reward_model_sglang: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 30 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK: "True" + NCCL_SHM_DISABLE: "1" + NCCL_P2P_DISABLE: "1" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install sglang-router==0.2.2 + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_dir ${HOME}/data/gsm8k + - name: Running sglang generative reward model tests on 8 L20 GPUs + run: | + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + ROLLOUT_NAME=sglang pytest -s -x tests/experimental/reward_loop/test_reward_model_genrm.py + - name: Running sglang discriminative reward model tests on 8 L20 GPUs + run: | + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + ROLLOUT_NAME=sglang pytest -s -x tests/experimental/reward_loop/test_reward_model_disrm.py + - name: Running sglang agent loop with reward manager tests on 8 L20 GPUs + run: | + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + ROLLOUT_NAME=sglang pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_standalone.py + - name: Running sglang agent loop with reward model colocate tests on 8 L20 GPUs + run: | + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + ROLLOUT_NAME=sglang pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_colocate.py + + cleanup: + runs-on: ubuntu-latest + needs: [setup, reward_model_sglang] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/reward_model_vllm.yml b/.github/workflows_old/reward_model_vllm.yml new file mode 100644 index 00000000000..aebde06984f --- /dev/null +++ b/.github/workflows_old/reward_model_vllm.yml @@ -0,0 +1,134 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. +# name: Check PR Title + +name: reward_model_vllm + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "verl/**/*.py" + # Entrypoints + - ".github/workflows/reward_model_vllm.yml" + - "tests/experimental/reward_loop/**" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + reward_model_vllm: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 30 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK: "True" + NCCL_SHM_DISABLE: "1" + NCCL_P2P_DISABLE: "1" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_dir ${HOME}/data/gsm8k + - name: Running vllm generative reward model tests on 8 L20 GPUs + run: | + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_reward_model_genrm.py + - name: Running vllm discriminative reward model tests on 8 L20 GPUs + run: | + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_reward_model_disrm.py + + - name: Running vllm agent loop with reward manager tests on 8 L20 GPUs + run: | + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_standalone.py + - name: Running vllm agent loop with reward model colocate tests on 8 L20 GPUs + run: | + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_colocate.py + + cleanup: + runs-on: ubuntu-latest + needs: [setup, reward_model_vllm] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/reward_model_vllm_ascend.yml b/.github/workflows_old/reward_model_vllm_ascend.yml new file mode 100644 index 00000000000..b57aa97c73b --- /dev/null +++ b/.github/workflows_old/reward_model_vllm_ascend.yml @@ -0,0 +1,113 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. +# name: Check PR Title + +name: reward_model_vllm_ascend + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "verl/**/*.py" + # Entrypoints + - ".github/workflows/reward_model_vllm_ascend.yml" + - "tests/experimental/reward_loop/**" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + reward_model_vllm_ascend: + if: github.repository_owner == 'verl-project' + runs-on: linux-aarch64-a2b3-8 + timeout-minutes: 60 # Increase this timeout value as needed + container: + image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest + options: >- + --shm-size 16g + env: + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - name: Check npu and CANN info + run: | + cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info + npu-smi info + - name: Check initial pip list from image + run: | + pip list + - name: Checkout verl-project/verl repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + clean: true + - name: Install the current repository + run: | + pip install -r requirements-npu.txt + pip install --no-deps -e .[test] + - name: Check final pip list + run: | + pip list + - name: Prepare weights + run: | + ln -s /root/.cache/models ~/models + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k --local_dir ${HOME}/data/gsm8k + - name: Running vllm generative reward model tests on 8 NPUs + run: | + ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_reward_model_genrm.py + - name: Running vllm discriminative reward model tests on 8 NPUs + run: | + ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_reward_model_disrm.py + - name: Running vllm agent loop with reward manager tests on 8 NPUs + run: | + ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_standalone.py + - name: Running vllm agent loop with reward model colocate tests on 8 NPUs + run: | + export HCCL_HOST_SOCKET_PORT_RANGE=auto + export HCCL_NPU_SOCKET_PORT_RANGE=auto + ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_colocate.py \ No newline at end of file diff --git a/.github/workflows_old/sanity.yml b/.github/workflows_old/sanity.yml new file mode 100644 index 00000000000..ac7532d2f04 --- /dev/null +++ b/.github/workflows_old/sanity.yml @@ -0,0 +1,108 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. +# name: Check PR Title + +name: sanity + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + - .github/workflows/sanity.yml + - "tests/special_sanity/**" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +jobs: + sanity: + runs-on: ubuntu-latest + timeout-minutes: 5 # Increase this timeout value as needed + strategy: + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install the current repository + run: | + pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu + pip3 install -r requirements.txt + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Run sanity test + run: | + pytest -s -x tests/special_sanity + - name: Run license test + run: | + python3 tests/special_sanity/check_license.py --directories . + - name: Assert naming convention + run: | + if grep -rIn --exclude-dir=.git --exclude-dir=.github --exclude-dir=venv --exclude-dir=__pycache__ 'veRL' .; then + echo "Please use verl instead of veRL in the codebase" + exit 1 + fi + - name: Assert SGLang naming convention + run: | + if grep -rIn --exclude-dir=.git --exclude-dir=.github --exclude-dir=venv --exclude-dir=__pycache__ --exclude=ascend_sglang_best_practices.rst -E 'Sglang|sgLang|sglAng|sglaNg|sglanG' .; then + echo "Please use SGLang or sglang as the formal name of SGLang rollout engine" + exit 1 + fi + - name: Validate test folder structure + run: python3 tests/special_sanity/validate_structure.py + - name: Assert documentation requirement for functions + run: python3 tests/special_sanity/validate_imported_docs.py + - name: Assert device api usage in verl/verl + run: python3 tests/special_sanity/check_device_api_usage.py --directory ./verl + - name: Assert documentation time info + run: python3 tests/special_sanity/check_docs_time_info.py + - name: Check docstrings for specified files + run: python3 tests/special_sanity/check_docstrings.py + - name: Check DataProto for specified folders + run: python3 tests/special_sanity/check_dataproto_usage.py -d ./verl/workers/engine diff --git a/.github/workflows_old/scorecard.yml b/.github/workflows_old/scorecard.yml new file mode 100644 index 00000000000..176d15ae2bd --- /dev/null +++ b/.github/workflows_old/scorecard.yml @@ -0,0 +1,66 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: "27 7 * * 1" + push: + branches: + - main + - v0.* + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + # Uncomment the permissions below if installing in a private repository. + # contents: read + # actions: read + + steps: + - name: "Checkout code" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecard on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results to GitHub's code scanning dashboard (optional). + # Commenting out will disable upload of results to your repo's Code Scanning dashboard + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 #v3.28.9 + with: + sarif_file: results.sarif diff --git a/.github/workflows_old/secrets_scan.yml b/.github/workflows_old/secrets_scan.yml new file mode 100644 index 00000000000..298ed16c668 --- /dev/null +++ b/.github/workflows_old/secrets_scan.yml @@ -0,0 +1,22 @@ +on: + push: + branches: + - main + - v0.* + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + - name: Secret Scanning + uses: trufflesecurity/trufflehog@7dc056a193116ba8d82154bf0549381c8fb8545c # v3.88.14 + with: + extra_args: --results=verified,unknown diff --git a/.github/workflows_old/sgl.yml b/.github/workflows_old/sgl.yml new file mode 100644 index 00000000000..bc0c0bb7f4a --- /dev/null +++ b/.github/workflows_old/sgl.yml @@ -0,0 +1,165 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: sgl + +on: + # workflow_dispatch: # Manual + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + paths: + - "**/*.py" + - .github/workflows/sgl.yml + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" # FSDP + - "!verl/workers/**/*dp_*.py" + # Megatron + - "!verl/workers/**/megatron_*.py" + # vLLM + - "!**/*vllm*" + + # Entrypoints + - ".github/workflows/sgl.yml" + - "tests/rollout/*sglang*" + - "tests/rollout/async_rollout_utils.py" + - "tests/workers/rollout/*interaction*" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + sgl: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 35 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: 1 + SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK: "True" + NCCL_SHM_DISABLE: "1" + NCCL_P2P_DISABLE: "1" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install cupy-cuda12x==13.6.0 pytest-asyncio + pip3 install hf_transfer fastmcp pytest-asyncio + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Test the latest SGLang Rollout async with agent loop + run: | + ROLLOUT_NAME=sglang pytest -svvv tests/experimental/agent_loop + + sgl_checkpoint_engine: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 35 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: 1 + SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK: "True" + NCCL_SHM_DISABLE: "1" + NCCL_P2P_DISABLE: "1" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install cupy-cuda12x==13.6.0 pytest-asyncio + pip3 install hf_transfer fastmcp pytest-asyncio + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + - name: Test SGLang ServerAdapter with Checkpoint Engine (NCCL) + run: | + ROLLOUT_NAME=sglang pytest -svvv tests/checkpoint_engine/test_special_server_adapter.py + + cleanup: + runs-on: ubuntu-latest + needs: [setup, sgl, sgl_checkpoint_engine] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/type-coverage-check.yml b/.github/workflows_old/type-coverage-check.yml new file mode 100644 index 00000000000..268f0c672f0 --- /dev/null +++ b/.github/workflows_old/type-coverage-check.yml @@ -0,0 +1,31 @@ +name: Type Annotation and Docstring Coverage + +on: + pull_request: + paths: + - '**/*.py' + - '.github/workflows/type-coverage-check.yml' + +jobs: + type-coverage-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # 🚨 Important: fetch full history so `origin/main` is available + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu + pip3 install -r requirements.txt + pip3 install --no-deps -e . + - name: Run type annotation coverage check + run: | + python3 tests/special_sanity/type_coverage_check.py + - name: Run docstring coverage check + run: | + python3 tests/special_sanity/check_api_docs.py verl diff --git a/.github/workflows_old/vllm.yml b/.github/workflows_old/vllm.yml new file mode 100644 index 00000000000..d358349f72c --- /dev/null +++ b/.github/workflows_old/vllm.yml @@ -0,0 +1,169 @@ +# # Tests layout + +# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: +# - `tests/trainer` for testing functionality related to `verl/trainer` +# - `tests/models` for testing functionality related to `verl/models` +# - ... + +# There are a few folders with `special_` prefix, created for special purposes: +# - `special_distributed`: unit tests that must run with multiple GPUs +# - `special_e2e`: end-to-end tests with training/generation scripts +# - `special_npu`: tests for NPUs +# - `special_sanity`: a suite of quick sanity tests +# - `special_standalone`: a set of test that are designed to run in dedicated environments + +# Accelerators for tests +# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. +# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. + +# # Workflow layout + +# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: +# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` +# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` +# 3. End-to-end tests: `e2e_*.yml` +# 4. Unit tests +# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` +# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. +# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when +# - new workflow yaml is added to `.github/workflows` +# - new tests are added to workflow mentioned in 2. + +name: vllm + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + - v0.* + pull_request: + branches: + - main + - v0.* + paths: + - "**/*.py" + # Other entrypoints + - "!examples/**" + - "!tests/**" + - "!verl/trainer/main_*.py" + - "!verl/trainer/fsdp_sft_trainer.py" + # FSDP + - "!verl/workers/**/*dp_*.py" + # Megatron + - "!verl/workers/**/megatron_*.py" + # SGLang + - "!**/*sglang*" + # Entrypoints + - ".github/workflows/vllm.yml" + - "tests/special_e2e/generation" + - "tests/workers/rollout" + - "verl/trainer/main_generation.py" + - "verl/trainer/config/generation.yaml" + +# Cancel jobs on the same ref if a new one is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Declare permissions just read content. +permissions: + contents: read + +env: + IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" + DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" + +jobs: + setup: + if: github.repository_owner == 'verl-project' + runs-on: ubuntu-latest + outputs: + runner-label: ${{ steps.create-runner.outputs.runner-label }} + mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} + steps: + - uses: actions/checkout@v4 + - id: create-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "create" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-image: "${{ env.IMAGE }}" + + vllm: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 35 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install --upgrade "transformers<5.0" + # - name: Download Model to Use + # run: | + # hf download Qwen/Qwen2.5-0.5B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-0.5B-Instruct + # hf download Qwen/Qwen2.5-1.5B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-1.5B-Instruct + # hf download Qwen/Qwen2.5-VL-3B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-VL-3B-Instruct + # hf download OldKingMeister/Qwen2.5-1.5B-Instruct-YaRN --local-dir ${HOME}/models/OldKingMeister/Qwen2.5-1.5B-Instruct-YaRN + # export HF_HUB_OFFLINE=1 + - name: Prepare gsm8k dataset + run: | + ray stop --force + python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k + - name: Test the latest vLLM Rollout async with agent loop + run: | + ROLLOUT_NAME=vllm pytest -svvv tests/experimental/agent_loop + - name: Test vllm server abort functionality + run: | + pytest tests/workers/rollout/rollout_vllm/test_vllm_abort.py -v -s + + vllm_checkpoint_engine: + needs: setup + runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] + timeout-minutes: 35 # Increase this timeout value as needed + env: + HTTP_PROXY: ${{ secrets.PROXY_HTTP }} + HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} + NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" + HF_ENDPOINT: "https://hf-mirror.com" + HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Install the current repository + run: | + pip3 install pytest-asyncio + pip3 install -r requirements-test.txt + pip3 install --no-deps -e . + pip3 install --upgrade "transformers<5.0" + pip3 install cupy-cuda12x==13.6.0 + - name: Test vLLM ServerAdapter with Checkpoint Engine (NCCL) + run: | + ROLLOUT_NAME=vllm pytest -svvv tests/checkpoint_engine/test_special_server_adapter.py + - name: Test bucketed weight transfer + run: | + pytest -svvv tests/utils/test_bucketed_weight_transfer.py + + cleanup: + runs-on: ubuntu-latest + needs: [setup, vllm, vllm_checkpoint_engine] + if: always() + steps: + - id: destroy-runner + uses: volcengine/vemlp-github-runner@v1 + with: + mode: "destroy" + faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" + mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" From d1786f5e5ec6052ef1bed5dd480575f97a6f04d8 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 2 Apr 2026 17:57:55 +0000 Subject: [PATCH 06/58] Disable CI --- .github/workflows/README.md | 73 ---- .github/workflows/check-pr-title.yml | 58 ---- .github/workflows/cpu_unit_tests.yml | 118 ------- .github/workflows/doc.yml | 101 ------ .github/workflows/docker-build-ascend-a2.yml | 84 ----- .github/workflows/docker-build-ascend-a3.yml | 84 ----- .github/workflows/e2e_ascend.yml | 166 --------- .github/workflows/e2e_fully_async_policy.yml | 170 ---------- .github/workflows/e2e_one_step_off_policy.yml | 171 ---------- .../e2e_one_step_off_policy_ascend.yml | 169 ---------- .../workflows/e2e_ppo_grpo_trainer_trtllm.yml | 287 ---------------- .github/workflows/e2e_ppo_trainer.yml | 78 ----- .../e2e_ppo_trainer_megatron_sglang.yml | 201 ----------- .../e2e_ppo_trainer_megatron_sglang_2.yml | 201 ----------- .../e2e_ppo_trainer_megatron_vllm.yml | 212 ------------ .../e2e_ppo_trainer_megatron_vllm_2.yml | 318 ------------------ ...e2e_ppo_trainer_megatron_vllm_2_ascend.yml | 233 ------------- .../workflows/e2e_ppo_trainer_veomni_vllm.yml | 153 --------- .github/workflows/e2e_sft_llm.yml | 153 --------- .github/workflows/e2e_sft_llm_ascend.yml | 160 --------- .github/workflows/e2e_sft_vlm.yml | 128 ------- .github/workflows/gpu_unit_tests.yml | 137 -------- .github/workflows/model.yml | 184 ---------- .github/workflows/model_ascend.yml | 137 -------- .github/workflows/nightly_ascend.yml | 174 ---------- .github/workflows/npu_unit_tests.yml | 126 ------- .github/workflows/precommit-autofix.yml | 52 --- .github/workflows/reward_model_sglang.yml | 134 -------- .github/workflows/reward_model_vllm.yml | 134 -------- .../workflows/reward_model_vllm_ascend.yml | 113 ------- .github/workflows/sanity.yml | 108 ------ .github/workflows/scorecard.yml | 66 ---- .github/workflows/secrets_scan.yml | 22 -- .github/workflows/sgl.yml | 165 --------- .github/workflows/type-coverage-check.yml | 31 -- .github/workflows/vllm.yml | 169 ---------- .github/workflows_old/pre-commit.yml | 41 --- 37 files changed, 5111 deletions(-) delete mode 100644 .github/workflows/README.md delete mode 100644 .github/workflows/check-pr-title.yml delete mode 100644 .github/workflows/cpu_unit_tests.yml delete mode 100644 .github/workflows/doc.yml delete mode 100644 .github/workflows/docker-build-ascend-a2.yml delete mode 100644 .github/workflows/docker-build-ascend-a3.yml delete mode 100644 .github/workflows/e2e_ascend.yml delete mode 100644 .github/workflows/e2e_fully_async_policy.yml delete mode 100644 .github/workflows/e2e_one_step_off_policy.yml delete mode 100644 .github/workflows/e2e_one_step_off_policy_ascend.yml delete mode 100644 .github/workflows/e2e_ppo_grpo_trainer_trtllm.yml delete mode 100644 .github/workflows/e2e_ppo_trainer.yml delete mode 100644 .github/workflows/e2e_ppo_trainer_megatron_sglang.yml delete mode 100644 .github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml delete mode 100644 .github/workflows/e2e_ppo_trainer_megatron_vllm.yml delete mode 100644 .github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml delete mode 100644 .github/workflows/e2e_ppo_trainer_megatron_vllm_2_ascend.yml delete mode 100644 .github/workflows/e2e_ppo_trainer_veomni_vllm.yml delete mode 100644 .github/workflows/e2e_sft_llm.yml delete mode 100644 .github/workflows/e2e_sft_llm_ascend.yml delete mode 100644 .github/workflows/e2e_sft_vlm.yml delete mode 100644 .github/workflows/gpu_unit_tests.yml delete mode 100644 .github/workflows/model.yml delete mode 100644 .github/workflows/model_ascend.yml delete mode 100644 .github/workflows/nightly_ascend.yml delete mode 100644 .github/workflows/npu_unit_tests.yml delete mode 100644 .github/workflows/precommit-autofix.yml delete mode 100644 .github/workflows/reward_model_sglang.yml delete mode 100644 .github/workflows/reward_model_vllm.yml delete mode 100644 .github/workflows/reward_model_vllm_ascend.yml delete mode 100644 .github/workflows/sanity.yml delete mode 100644 .github/workflows/scorecard.yml delete mode 100644 .github/workflows/secrets_scan.yml delete mode 100644 .github/workflows/sgl.yml delete mode 100644 .github/workflows/type-coverage-check.yml delete mode 100644 .github/workflows/vllm.yml delete mode 100644 .github/workflows_old/pre-commit.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md deleted file mode 100644 index d83c87b2e71..00000000000 --- a/.github/workflows/README.md +++ /dev/null @@ -1,73 +0,0 @@ -### Adding a New Workflow - -When adding a new workflow for continuous integration (CI), you have two runner options: a fixed runner or a machine from the vemlp. - -- **Fixed Runner**: To use a fixed runner, specify it in your workflow using the `runs-on` keyword, like `runs-on: [L20x8]`. -- **Vemlp Runner**: Opting for a Vemlp machine allows you to launch tasks elastically. - -Here is a template to assist you. This template is designed for using Vemlp machines. Currently, for each workflow, you need to create a `setup` and a `cleanup` job. When using this template, the main parts you need to modify are the `IMAGE` environment variable and the specific `job steps`. - -```yaml -name: Your Default Workflow - -on: - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - - ".github/workflows/template.yml" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -permissions: - contents: read - -env: - IMAGE: "your vemlp image" # e.g. "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" - DYNAMIC_RUNNER_URL: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" # public veFaas api - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - task-id: ${{ steps.create-runner.outputs.task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_URL }}" - image: "${{ env.DEFAULT_IMAGE }}" - - your_job: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'default-runner' }}"] - steps: - xxxx # your jobs - - cleanup: - runs-on: ubuntu-latest - needs: [setup, your_job] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_URL }}" - task-id: "${{ needs.setup.outputs.task-id }}" -``` - -### Model and Dataset -To avoid CI relies on network, we pre-download dataset on a NFS on the CI machine. The path for models are \${HOME}/models and the path for dataset is \${HOME}/models/hf_data. \ No newline at end of file diff --git a/.github/workflows/check-pr-title.yml b/.github/workflows/check-pr-title.yml deleted file mode 100644 index 948ce5e3f01..00000000000 --- a/.github/workflows/check-pr-title.yml +++ /dev/null @@ -1,58 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - - -on: - pull_request: - types: [opened, edited, synchronize] - -jobs: - check-title: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Run PR title checker - run: python3 tests/special_sanity/check_pr_title.py - env: - PR_TITLE: ${{ github.event.pull_request.title }} - - - name: Run PR description checker - run: python3 tests/special_sanity/check_pr_description.py - env: - PR_TITLE: ${{ github.event.pull_request.title }} - GITHUB_EVENT_PATH: ${{ github.event_path }} diff --git a/.github/workflows/cpu_unit_tests.yml b/.github/workflows/cpu_unit_tests.yml deleted file mode 100644 index 48ce123bc07..00000000000 --- a/.github/workflows/cpu_unit_tests.yml +++ /dev/null @@ -1,118 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: cpu_unit_tests - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - - .github/workflows/cpu_unit_tests.yml - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - cpu_unit_tests: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 20 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - TORCH_COMPILE_DISABLE: 1 - TORCHINDUCTOR_DISABLE: 1 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install --upgrade "transformers>=5.0.0" - - name: Download datasets - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k - - name: Running CPU unit tests - run: | - echo '[pytest]' > pytest.ini - echo 'python_files = *_on_cpu.py' >> pytest.ini - pytest -s -x --asyncio-mode=auto tests/ - cleanup: - runs-on: ubuntu-latest - needs: [setup, cpu_unit_tests] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml deleted file mode 100644 index aa4a713deac..00000000000 --- a/.github/workflows/doc.yml +++ /dev/null @@ -1,101 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - - -name: doc_test - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - - "docs/**" - - .github/workflows/doc.yml - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read # for checkout - pages: write # for deploy-pages - id-token: write # for deploy-pages - -jobs: - doc_test: - runs-on: ubuntu-latest - timeout-minutes: 5 # Increase this timeout value as needed - strategy: - matrix: - python-version: ["3.10"] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: ${{ matrix.python-version }} - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip install -r docs/requirements-docs.txt - - - name: Run doc make html - run: | - cd docs - make clean - make html SPHINXOPTS="--keep-going -w _build/sphinx.log" - if grep -q ": ERROR:" _build/sphinx.log; then - echo "🚨 Sphinx doc build contained ERRORs - see _build/sphinx.log" - exit 1 - fi - if grep -q "WARNING: document isn't included in any toctree" _build/sphinx.log; then - echo "🚨 Sphinx doc build contained WARNING. Please include newly added docs in index.rst. See _build/sphinx.log for details" - exit 1 - fi - if grep -q "WARNING: Inline emphasis" _build/sphinx.log; then - echo "🚨 Sphinx doc build contained WARNING. Please check inline emphasis is correct. See _build/sphinx.log for details" - exit 1 - fi - if grep -q "WARNING: Definition list ends without a blank line" _build/sphinx.log; then - echo "🚨 Sphinx doc build contained WARNING. Please check if the indentation is correct. See _build/sphinx.log for details" - exit 1 - fi diff --git a/.github/workflows/docker-build-ascend-a2.yml b/.github/workflows/docker-build-ascend-a2.yml deleted file mode 100644 index 76540a53c32..00000000000 --- a/.github/workflows/docker-build-ascend-a2.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: docker-build-ascend-a2 - -on: - workflow_dispatch: - push: - branches: ["main"] - paths: - - "docker/ascend/Dockerfile.ascend_8.5.0_a2" - - ".github/workflows/docker-build-ascend-a2.yml" - release: - types: [published] - schedule: - - cron: "0 16 * * *" - -jobs: - build-ascend-image-a2: - if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'verl-project' }} - runs-on: ubuntu-latest - concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-build-ascend-image-a2 - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - steps: - - name: Remove unnecessary parts in github actions runners to free up disk space - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: true - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Get base image name and tag - id: base_image - run: | - BASE_IMAGE_FULL=$(grep '^FROM' ./docker/ascend/Dockerfile.ascend_8.5.0_a2 | head -1 | cut -d' ' -f2) - echo "Base image full: $BASE_IMAGE_FULL" - BASE_IMAGE_TAG=$(echo "$BASE_IMAGE_FULL" | cut -d':' -f2) - echo "Base image tag: $BASE_IMAGE_TAG" - NEW_IMAGE_NAME="verl-$BASE_IMAGE_TAG" - echo "New image name: $NEW_IMAGE_NAME" - echo "base_image_tag=$BASE_IMAGE_TAG" >> "$GITHUB_OUTPUT" - echo "new_image_name=$NEW_IMAGE_NAME" >> "$GITHUB_OUTPUT" - - - name: Get image tag - id: version - run: | - BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///g' | sed 's/[^a-zA-Z0-9._-]/_/g') - if [ "${{ github.event_name }}" = "release" ]; then - echo "tag=${{ steps.base_image.outputs.new_image_name }}-${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT" - elif [ "$BRANCH_NAME" = "main" ]; then - echo "tag=${{ steps.base_image.outputs.new_image_name }}-latest" >> "$GITHUB_OUTPUT" - fi - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Quay.io - uses: docker/login-action@v3 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} - - - name: Clean Docker cache before build - run: | - docker system prune -a -f --volumes || true - - - name: Build and push images Quay - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64,linux/arm64 - file: ./docker/ascend/Dockerfile.ascend_8.5.0_a2 - push: true - tags: | - quay.io/ascend/verl:${{ steps.version.outputs.tag }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - BUILDKIT_INLINE_CACHE=1 diff --git a/.github/workflows/docker-build-ascend-a3.yml b/.github/workflows/docker-build-ascend-a3.yml deleted file mode 100644 index 6549387fddc..00000000000 --- a/.github/workflows/docker-build-ascend-a3.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: docker-build-ascend-a3 - -on: - workflow_dispatch: - push: - branches: ["main"] - paths: - - "docker/ascend/Dockerfile.ascend_8.5.0_a3" - - ".github/workflows/docker-build-ascend-a3.yml" - release: - types: [published] - schedule: - - cron: "0 19 * * *" - -jobs: - build-ascend-image-a3: - if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'verl-project' }} - runs-on: ubuntu-latest - concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-build-ascend-image-a3 - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - steps: - - name: Remove unnecessary parts in github actions runners to free up disk space - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: true - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Get base image name and tag - id: base_image - run: | - BASE_IMAGE_FULL=$(grep '^FROM' ./docker/ascend/Dockerfile.ascend_8.5.0_a3 | head -1 | cut -d' ' -f2) - echo "Base image full: $BASE_IMAGE_FULL" - BASE_IMAGE_TAG=$(echo "$BASE_IMAGE_FULL" | cut -d':' -f2) - echo "Base image tag: $BASE_IMAGE_TAG" - NEW_IMAGE_NAME="verl-$BASE_IMAGE_TAG" - echo "New image name: $NEW_IMAGE_NAME" - echo "base_image_tag=$BASE_IMAGE_TAG" >> "$GITHUB_OUTPUT" - echo "new_image_name=$NEW_IMAGE_NAME" >> "$GITHUB_OUTPUT" - - - name: Get image tag - id: version - run: | - BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///g' | sed 's/[^a-zA-Z0-9._-]/_/g') - if [ "${{ github.event_name }}" = "release" ]; then - echo "tag=${{ steps.base_image.outputs.new_image_name }}-${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT" - elif [ "$BRANCH_NAME" = "main" ]; then - echo "tag=${{ steps.base_image.outputs.new_image_name }}-latest" >> "$GITHUB_OUTPUT" - fi - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Quay.io - uses: docker/login-action@v3 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} - - - name: Clean Docker cache before build - run: | - docker system prune -a -f --volumes || true - - - name: Build and push images Quay - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64,linux/arm64 - file: ./docker/ascend/Dockerfile.ascend_8.5.0_a3 - push: true - tags: | - quay.io/ascend/verl:${{ steps.version.outputs.tag }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - BUILDKIT_INLINE_CACHE=1 diff --git a/.github/workflows/e2e_ascend.yml b/.github/workflows/e2e_ascend.yml deleted file mode 100644 index d4ea77ad143..00000000000 --- a/.github/workflows/e2e_ascend.yml +++ /dev/null @@ -1,166 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_ascend - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - paths: - - ".github/workflows/e2e_ascend.yml" - - "examples/data_preprocess/**" - - "examples/grpo_trainer/**" - - "examples/ppo_trainer/**" - - "examples/sft/**" - - "verl/experimental/one_step_off_policy/**" - - "tests/special_npu/**" - - "tests/special_sanity/check_device_api_usage.py" - - "verl/**" - - "pyproject.toml" - - "requirements-npu.txt" - - "setup.py" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -permissions: - contents: read - -jobs: - llm_rl_job: - if: github.repository_owner == 'verl-project' - name: E2E Ascend testing for RL training scenarios of LLM models - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 120 - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout volcengine/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install -e . - - name: Check final pip list - run: | - pip list - - name: Preprocess gsm8k dataset - run: | - python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k - - name: Running gsm8k e2e training tests with PPO on ASCEND NPU (FSDP backend) - run: | - ray stop --force - bash tests/special_npu/run_qwen3_06b_ppo.sh - rm -rf $HOME/ckpts - - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (FSDP backend) - run: | - ray stop --force - bash tests/special_npu/run_qwen2_5_05b_grpo.sh - rm -rf $HOME/ckpts - - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (MindSpeed backend) - run: | - ray stop --force - USE_DIST_CKPT=True bash tests/special_npu/run_qwen2_5_05b_grpo_mindspeed.sh - rm -rf $HOME/dist_ckpt/qwen2_5_05b_grpo_mindspeed - rm -rf $HOME/ckpts - - name: Running gsm8k e2e training tests with GRPO on ASCEND NPU (MindSpeed backend, MoE Model) - run: | - ray stop --force - USE_DIST_CKPT=True USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen3moe_minimal.json DUMMY_MODEL_PATH=$HOME/dist_ckpt/qwen3_30b_grpo_mindspeed bash tests/special_npu/run_qwen3_30b_grpo_mindspeed.sh - - name: Running the E2E test with fully_async_policy algorithm (FSDP2) - run: | - ray stop --force - bash tests/special_npu/run_fully_async_policy.sh - - vlm_rl_job: - if: github.repository_owner == 'verl-project' - name: E2E Ascend testing for RL training scenarios of VLM models - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 120 - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout volcengine/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install -e . - - name: Check final pip list - run: | - pip list - - name: Preprocess geo3k dataset - run: | - python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k - - name: Running geo3k e2e training tests with GRPO on ASCEND NPU - run: | - ray stop --force - bash tests/special_npu/run_qwen2_5_vl_3b_npu.sh - rm -rf $HOME/ckpts diff --git a/.github/workflows/e2e_fully_async_policy.yml b/.github/workflows/e2e_fully_async_policy.yml deleted file mode 100644 index a46be304814..00000000000 --- a/.github/workflows/e2e_fully_async_policy.yml +++ /dev/null @@ -1,170 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_fully_async_policy - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - - "!**/*.md" - - "!**/*.sh" - # Other entrypoints - - "!examples/*trainer*" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - - "verl/experimental/fully_async_policy" - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - - "!**/*.md" - - "!**/*.sh" - # Other entrypoints - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # Home - - "verl/experimental/fully_async_policy" - # Entrypoints - - ".github/workflows/e2e_fully_async_policy.yml" - - "examples/data_preprocess/gsm8k.py" - - "tests/special_e2e/run_fully_async_policy.sh" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - # Test FSDP2 strategy - e2e_fully_async_policy_fsdp2: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 10 # Increase timeout for async training - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - ACTOR_STRATEGY: "fsdp2" - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install cupy-cuda12x==13.6.0 - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Running the E2E test with fully_async_policy algorithm (FSDP2) - run: | - ray stop --force - bash tests/special_e2e/run_fully_async_policy.sh - - # Test Megatron strategy - e2e_fully_async_policy_megatron: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 10 # Increase timeout for async training - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - ACTOR_STRATEGY: "megatron" - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install cupy-cuda12x==13.6.0 - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Running the E2E test with fully_async_policy algorithm (Megatron) - run: | - ray stop --force - bash tests/special_e2e/run_fully_async_policy.sh - - cleanup: - runs-on: ubuntu-latest - needs: [setup, e2e_fully_async_policy_fsdp2] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/e2e_one_step_off_policy.yml b/.github/workflows/e2e_one_step_off_policy.yml deleted file mode 100644 index de3f8df5c1e..00000000000 --- a/.github/workflows/e2e_one_step_off_policy.yml +++ /dev/null @@ -1,171 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_one_step_off_policy - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - - "!**/*.md" - - "!**/*.sh" - # Other entrypoints - - "!examples/*trainer*" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - - "verl/experimental/one_step_off_policy" - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - - "!**/*.md" - - "!**/*.sh" - # Other entrypoints - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # Home - - "verl/experimental/one_step_off_policy" - # Entrypoints - - ".github/workflows/e2e_one_step_off_policy.yml" - - "examples/data_preprocess/gsm8k.py" - - "tests/special_e2e/run_one_step_off_policy.sh" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - # Test FSDP2 strategy - e2e_one_step_off_policy_fsdp2: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 10 # Increase timeout for async training - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - ACTOR_STRATEGY: "fsdp2" - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install cupy-cuda12x==13.6.0 - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Running the E2E test with one_step_off_policy algorithm (FSDP2) - run: | - ray stop --force - bash tests/special_e2e/run_one_step_off_policy.sh - - # Test Megatron strategy - e2e_one_step_off_policy_megatron: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 10 # Increase timeout for async training - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - ACTOR_STRATEGY: "megatron" - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install cupy-cuda12x==13.6.0 - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Running the E2E test with one_step_off_policy algorithm (Megatron) - run: | - ray stop --force - bash tests/special_e2e/run_one_step_off_policy.sh - - cleanup: - runs-on: ubuntu-latest - needs: - [setup, e2e_one_step_off_policy_fsdp2, e2e_one_step_off_policy_megatron] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/e2e_one_step_off_policy_ascend.yml b/.github/workflows/e2e_one_step_off_policy_ascend.yml deleted file mode 100644 index 77ed29b4e03..00000000000 --- a/.github/workflows/e2e_one_step_off_policy_ascend.yml +++ /dev/null @@ -1,169 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_one_step_off_policy_ascend - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - - "!**/*.md" - - "!**/*.sh" - # Other entrypoints - - "!examples/*trainer*" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - - "verl/experimental/one_step_off_policy" - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - - "!**/*.md" - - "!**/*.sh" - # Other entrypoints - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # Home - - "verl/experimental/one_step_off_policy" - # Entrypoints - - ".github/workflows/e2e_one_step_off_policy_ascend.yml" - - "examples/data_preprocess/gsm8k.py" - - "tests/special_npu/run_one_step_off_policy.sh" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -jobs: - # Test FSDP2 strategy - e2e_one_step_off_policy_fsdp2_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 60 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - ACTOR_STRATEGY: "fsdp2" - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout verl-project/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e . - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Prepare GSM8K dataset - run: | - python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k - - name: Running the E2E test with one_step_off_policy algorithm (FSDP2) - run: | - ray stop --force - bash tests/special_npu/run_one_step_off_policy.sh - - # Test Megatron strategy - e2e_one_step_off_policy_megatron_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 60 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - ACTOR_STRATEGY: "megatron" - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout verl-project/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e . - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Prepare GSM8K dataset - run: | - python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k - - name: Running the E2E test with one_step_off_policy algorithm (Megatron) - run: | - ray stop --force - bash tests/special_npu/run_one_step_off_policy.sh diff --git a/.github/workflows/e2e_ppo_grpo_trainer_trtllm.yml b/.github/workflows/e2e_ppo_grpo_trainer_trtllm.yml deleted file mode 100644 index 61a19d43419..00000000000 --- a/.github/workflows/e2e_ppo_grpo_trainer_trtllm.yml +++ /dev/null @@ -1,287 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_ppo_trainer_megatron_trtllm - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch. - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!verl/trainer/fsdp_sft_trainer.py" - # Recipes - - "!recipe/**" - # FSDP - - "!verl/workers/**/*dp_*.py" - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!docker/**" - # Docs - - "!**/*.md" - - "!docs/**" - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # Recipes - - "!recipe/**" - # FSDP - - "!verl/workers/**/*dp_*.py" - # Entrypoints - - "verl/workers/rollout/trtllm_rollout/**" - - "tests/workers/rollout/rollout_trtllm/**" - - ".github/workflows/e2e_ppo_grpo_trainer_trtllm.yml" - - "examples/data_preprocess/gsm8k.py" - - "examples/data_preprocess/geo3k.py" - - "examples/data_preprocess/dapo_multiturn_w_tool.py" - - "examples/data_preprocess/aime2024_multiturn_w_tool.py" - - "examples/grpo_trainer/run_qwen2-7b_math_trtllm.sh" - - "examples/grpo_trainer/run_qwen2-7b_math_megatron_trtllm.sh" - - "examples/grpo_trainer/run_qwen3-30b_dapo_megatron_fp8_trtllm.sh" - # add back when ppo flow is ready - # - "tests/special_e2e/run_ppo_trainer_megatron.sh" - # - "verl/trainer/main_ppo.py" - # - "verl/trainer/config/ppo_megatron_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:trtllm1.3.0rc4" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - trtllm_unit_tests: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 30 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install pytest-asyncio - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Run TRTLLM unit tests - run: | - export TRTLLM_TEST_MODEL_PATH_ROOT="${HOME}/models" - ray stop --force - pytest -v -s \ - tests/workers/rollout/rollout_trtllm/test_adapter.py \ - tests/workers/rollout/rollout_trtllm/test_async_server.py \ - tests/workers/rollout/rollout_trtllm/test_trtllm_rollout_utils.py - - e2e_grpo_trainer_fsdp-qwen2: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 30 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_save_dir ${PWD}/data/gsm8k - - name: Running GSM8K E2E training tests with FSDP on 8 L20 GPUs (Qwen) - run: | - ray stop --force - DATADIR=${HOME}/data \ - bash examples/grpo_trainer/run_qwen2-7b_math_trtllm.sh 2 \ - trainer.total_training_steps=1 \ - data.train_files="['${PWD}/data/gsm8k/train.parquet']" \ - data.val_files="['${PWD}/data/gsm8k/test.parquet']" \ - trainer.logger='["console"]' \ - actor_rollout_ref.model.path="${HOME}/models/Qwen/Qwen2.5-0.5B-Instruct" - - name: clean up - run: | - rm -rf checkpoints - - e2e_grpo_trainer_megatron-qwen2: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 30 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_save_dir ${PWD}/data/gsm8k - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen) - run: | - ray stop --force - DATADIR=${HOME}/data \ - ACTOR_TP=2 \ - bash examples/grpo_trainer/run_qwen2-7b_math_megatron_trtllm.sh 2 \ - trainer.total_training_steps=1 \ - data.train_files="['${PWD}/data/gsm8k/train.parquet']" \ - data.val_files="['${PWD}/data/gsm8k/test.parquet']" \ - trainer.logger='["console"]' \ - actor_rollout_ref.model.path="${HOME}/models/Qwen/Qwen2.5-0.5B-Instruct" - - name: clean up - run: | - rm -rf checkpoints - e2e_grpo_trainer_fsdp-vlm: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 30 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install qwen_vl_utils - pip3 install mathruler - - name: Prepare GEO3K dataset - run: | - python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k --local_save_dir ${PWD}/data/geo3k - - name: Running GEO3K E2E training tests with FSDP on 8 L20 GPUs (VLM) - run: | - ray stop --force - DATADIR=${HOME}/data \ - bash examples/grpo_trainer/run_qwen2_5_vl_3b_trtllm.sh 2 \ - trainer.total_training_steps=1 \ - data.train_files="['${PWD}/data/geo3k/train.parquet']" \ - data.val_files="['${PWD}/data/geo3k/test.parquet']" \ - trainer.logger='["console"]' \ - actor_rollout_ref.model.path="${HOME}/models/Qwen/Qwen3-VL-2B-Instruct" - - name: clean up - run: | - rm -rf checkpoints - - name: Prepare DAPO-Math-17k and AIME-2024 datasets (data_preprocess) - run: | - python3 examples/data_preprocess/dapo_multiturn_w_tool.py --local_save_dir ${PWD}/data/dapo-math-17k - python3 examples/data_preprocess/aime2024_multiturn_w_tool.py --local_save_dir ${PWD}/data/aime-2024 - - name: Running DAPO E2E with FP8 TRT-LLM rollout (Qwen3-0.6B) - run: | - ray stop --force - export INFER_TP=2 ACTOR_TP=2 ACTOR_PP=2 ACTOR_VPP=2 ACTOR_EP=1 ACTOR_CP=2 REF_TP=2 REF_PP=2 REF_VPP=2 REF_EP=1 REF_CP=2 GEN_MOE_TP=null GEN_MOE_EP=null - export NNODES=1 GPUS_PER_NODE=8 TRTLLM_MOE_BACKEND=CUTLASS - export DATA_DIR=${PWD} DAPO_MATH_TRAIN=${PWD}/data/dapo-math-17k/train.parquet AIME_VAL=${PWD}/data/aime-2024/train.parquet MODEL_PATH=${HOME}/models/Qwen/Qwen3-0.6B - bash examples/grpo_trainer/run_qwen3-30b_dapo_megatron_fp8_trtllm.sh \ - reward_model.reward_kwargs.overlong_buffer_cfg.len=258 \ - reward_model.reward_kwargs.max_resp_len=512 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.train_batch_size=32 \ - actor_rollout_ref.rollout.n=4 \ - actor_rollout_ref.rollout.max_num_seqs=16 \ - actor_rollout_ref.rollout.max_num_batched_tokens=1024 \ - actor_rollout_ref.rollout.max_model_len=1024 \ - actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=False \ - actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=False \ - trainer.total_training_steps=1 \ - trainer.logger='["console"]' - - name: clean up - run: | - rm -rf checkpoints - - cleanup: - runs-on: ubuntu-latest - needs: [setup, trtllm_unit_tests, e2e_grpo_trainer_fsdp-qwen2, e2e_grpo_trainer_megatron-qwen2, e2e_grpo_trainer_fsdp-vlm] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/e2e_ppo_trainer.yml b/.github/workflows/e2e_ppo_trainer.yml deleted file mode 100644 index 357f0aa6bb6..00000000000 --- a/.github/workflows/e2e_ppo_trainer.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: e2e_ppo_trainer - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!verl/trainer/fsdp_sft_trainer.py" - - # Megatron - - "!verl/workers/**/megatron_*.py" - - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!**/*.md" - - "!docker/**" - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # Docs - - "!docs/**" - - # Megatron - - "!verl/workers/**/megatron_*.py" - # Entrypoints - - ".github/workflows/e2e_ppo_trainer.yml" - - "examples/data_preprocess/gsm8k.py" - - "examples/data_preprocess/geo3k.py" - - "tests/special_e2e/ppo_trainer" - - "verl/trainer/main_ppo.py" - - "verl/trainer/config/ppo_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -jobs: - pre_commit_for_ppo: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.12"] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: ${{ matrix.python-version }} - - name: Install the current repository - run: | - pip install pre-commit hydra-core - pip3 install --no-deps -e . - - name: Set ruff --output-format=github - run: | - sed -i 's/--output-format=full/--output-format=github/' .pre-commit-config.yaml - git add .pre-commit-config.yaml - - uses: pre-commit/action@v3.0.1 - with: - extra_args: "" # Overriding default "--all-files" - diff --git a/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml b/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml deleted file mode 100644 index 5a8ef80432e..00000000000 --- a/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml +++ /dev/null @@ -1,201 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_ppo_trainer_megatron_sglang - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch. - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!verl/trainer/fsdp_sft_trainer.py" # FSDP - - "!verl/workers/**/*dp_*.py" - - "!verl/utils/fsdp_utils.py" - - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" - - "!verl/model_merger/fsdp_model_merger.py" - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!docker/**" - # Docs - - "!**/*.md" - - "!docs/**" - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" # FSDP - - "!verl/workers/**/*dp_*.py" - - "!verl/utils/fsdp_utils.py" - - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" - - "!verl/model_merger/fsdp_model_merger.py" - # Entrypoints - - "verl/worksers/rollout/sglang_rollout/*" - - ".github/workflows/e2e_ppo_trainer_megatron_sglang.yml" - - "examples/data_preprocess/gsm8k.py" - - "examples/data_preprocess/geo3k.py" - - "tests/special_e2e/run_ppo_trainer_megatron.sh" - - "verl/trainer/main_ppo.py" - - "verl/trainer/config/ppo_megatron_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - e2e_ppo_trainer_megatron-deepseek: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 60 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - ENGINE: sglang - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) - run: | - ray stop --force - OPTIM_MEMORY_EFFICIENT=True ENGINE=sglang SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (DeepSeek) - run: | - ray stop --force - export VLLM_USE_V1=1 - ray start --head - ENGINE=sglang MODE=async RESUME_MODE=auto MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct TOTAL_TRAIN_STEPS=2 bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: Profiling GRPO GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Deepseek) - run: | - ray stop --force - PROFILE_ENABLE=True ENGINE=sglang ADV_ESTIMATOR=grpo USE_DYNAMIC_BSZ=False MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct bash tests/special_e2e/run_ppo_trainer_megatron.sh - if [ -z "$( ls -A '/tmp/ray/session_latest/logs/nsight/' )" ]; then - echo "[ERROR] not found any profiling files" - exit 1 - else - echo "[SUCCESS] profile success" - fi - - name: clean up - run: | - rm -rf checkpoints - - # Qwen3-0.6B: dense, tie_word_embeddings=True - e2e_ppo_trainer_megatron-qwen3: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 60 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - ENGINE: sglang - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen3) testing learning rate scheduler - run: | - ray stop --force - ALL_OFFLOAD=True VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 LR_WARMUP_STEPS=1 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with FP8 rollout - run: | - ray stop --force - export VLLM_USE_V1=1 - ROLLOUT_QUANTIZATION=fp8 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: clean up - run: | - rm -rf checkpoints - - cleanup: - runs-on: ubuntu-latest - needs: - [setup, e2e_ppo_trainer_megatron-deepseek, e2e_ppo_trainer_megatron-qwen3] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml b/.github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml deleted file mode 100644 index ba9d3b23545..00000000000 --- a/.github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml +++ /dev/null @@ -1,201 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_ppo_trainer_megatron_sglang_2 - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch. - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!verl/trainer/fsdp_sft_trainer.py" # FSDP - - "!verl/workers/**/*dp_*.py" - - "!verl/utils/fsdp_utils.py" - - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" - - "!verl/model_merger/fsdp_model_merger.py" - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!docker/**" - # Docs - - "!**/*.md" - - "!docs/**" - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" # FSDP - - "!verl/workers/**/*dp_*.py" - - "!verl/utils/fsdp_utils.py" - - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" - - "!verl/model_merger/fsdp_model_merger.py" - # Entrypoints - - "verl/worksers/rollout/sglang_rollout/*" - - ".github/workflows/e2e_ppo_trainer_megatron_sglang.yml" - - "examples/data_preprocess/gsm8k.py" - - "examples/data_preprocess/geo3k.py" - - "tests/special_e2e/run_ppo_trainer_megatron.sh" - - "verl/trainer/main_ppo.py" - - "verl/trainer/config/ppo_megatron_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - e2e_ppo_trainer_fsdp_sglang: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 40 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Prepare gsm8k dataset - run: | - ray stop --force - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm and save ckpt - run: | - ray stop --force - ENGINE=sglang bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - e2e_ppo_trainer_fsdp-qwen2_5vl-3b: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 60 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - # Geo3k - - name: Prepare GEO3K dataset - run: | - ray stop --force - python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ - - name: Running GEO3K VLM E2E training tests on 8 L20 GPUs with rmpad using function rm - run: | - ray stop --force - TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ - MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ - MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ - ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ - ENGINE=sglang ROLLOUT_MODE=async GPU_MEMORY_UTILIZATION=0.6 ACTOR_FSDP_PARAM_OFFLOAD=True \ - ACTOR_FSDP_OPTIMIZER_OFFLOAD=True REF_FSDP_PARAM_OFFLOAD=True \ - bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Running GEO3K VLM E2E with rmpad using torch fused kernel (Qwen2.5-VL) - run: | - ray stop --force - FUSED_KERNELS=True TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ - MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ - MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ - ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ - ENGINE=sglang ROLLOUT_MODE=async GPU_MEMORY_UTILIZATION=0.6 ACTOR_FSDP_PARAM_OFFLOAD=True \ - ACTOR_FSDP_OPTIMIZER_OFFLOAD=True REF_FSDP_PARAM_OFFLOAD=True \ - bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Running GEO3K VLM E2E with rmpad using triton fused kernel (Qwen2.5-VL) - run: | - ray stop --force - FUSED_KERNELS=True FUSED_KERNEL_BACKEND=triton \ - TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ - MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ - MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ - ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ - ENGINE=sglang ROLLOUT_MODE=async GPU_MEMORY_UTILIZATION=0.6 ACTOR_FSDP_PARAM_OFFLOAD=True \ - ACTOR_FSDP_OPTIMIZER_OFFLOAD=True REF_FSDP_PARAM_OFFLOAD=True \ - bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - cleanup: - runs-on: ubuntu-latest - needs: - [setup, e2e_ppo_trainer_fsdp-qwen2_5vl-3b, e2e_ppo_trainer_fsdp_sglang] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/e2e_ppo_trainer_megatron_vllm.yml b/.github/workflows/e2e_ppo_trainer_megatron_vllm.yml deleted file mode 100644 index d9fa832a56b..00000000000 --- a/.github/workflows/e2e_ppo_trainer_megatron_vllm.yml +++ /dev/null @@ -1,212 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_ppo_trainer_megatron_vllm - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch. - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!verl/trainer/fsdp_sft_trainer.py" - # FSDP - - "!verl/workers/**/*dp_*.py" - - "!verl/utils/fsdp_utils.py" - - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" - - "!verl/model_merger/fsdp_model_merger.py" - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!docker/**" - # Docs - - "!**/*.md" - - "!docs/**" - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # FSDP - - "!verl/workers/**/*dp_*.py" - - "!verl/utils/fsdp_utils.py" - - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" - - "!verl/model_merger/fsdp_model_merger.py" - # Entrypoints - - ".github/workflows/e2e_ppo_trainer_megatron_vllm.yml" - - "examples/data_preprocess/gsm8k.py" - - "examples/data_preprocess/geo3k.py" - - "tests/special_e2e/run_ppo_trainer_megatron.sh" - - "verl/trainer/main_ppo.py" - - "verl/trainer/config/ppo_megatron_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - # deepseek-ai/deepseek-coder-1.3b-instruct: dense, tie_word_embeddings=False - e2e_ppo_trainer_megatron-deepseek: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 60 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps --force-reinstall . - pip3 install mbridge - pip3 install math-verify - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - # Full training save&load - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron, use mbridge e2e to pre-load and save (Deepseek) - run: | - ray stop --force - ALL_OFFLOAD=True SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 USE_MBRIDGE=True USE_DIST_CKPT=False \ - bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron, use mbridge e2e to pre-load and save (Deepseek) - run: | - ray stop --force - RESUME_MODE=auto MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct TOTAL_TRAIN_STEPS=2 SAVE_FREQ=1 COMMON_PP=4 COMMON_VPP=null COMMON_CP=1 USE_MBRIDGE=True USE_DIST_CKPT=False \ - bash tests/special_e2e/run_ppo_trainer_megatron.sh - # LoRA training save&load - - name: clean up and install Megatron-Bridge - run: | - rm -rf checkpoints - pip3 install git+https://github.com/NVIDIA-NeMo/Megatron-Bridge.git@83a7c11 --no-deps --no-build-isolation - pip3 install git+https://github.com/NVIDIA/Megatron-LM.git@5455f0a --no-deps --no-build-isolation - pip3 install "nvidia-modelopt[torch]>=0.37.0" transformers==4.57.1 - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron, use Megatron-Bridge LoRA e2e to pre-load and save (Deepseek) - run: | - ray stop --force - ALL_OFFLOAD=True SAVE_FREQ=1 MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct COMMON_PP=4 LORA_RANK=8 COMMON_VPP=null COMMON_CP=1 USE_MBRIDGE=True VANILLA_MBRIDGE=False VALUE_VANILLA_MBRIDGE=False USE_DIST_CKPT=False \ - bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron, use Megatron-Bridge LoRA e2e to pre-load and save (Deepseek) - run: | - ray stop --force - RESUME_MODE=auto MODEL_ID=deepseek-ai/deepseek-coder-1.3b-instruct TOTAL_TRAIN_STEPS=2 SAVE_FREQ=1 COMMON_PP=4 LORA_RANK=8 COMMON_VPP=null COMMON_CP=1 USE_MBRIDGE=True VANILLA_MBRIDGE=False VALUE_VANILLA_MBRIDGE=False USE_DIST_CKPT=False \ - bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: clean up - run: | - rm -rf checkpoints - - # Qwen3-0.6B: dense, tie_word_embeddings=True - e2e_ppo_trainer_megatron-qwen3: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 60 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install math-verify - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron (Qwen3) testing learning rate scheduler - run: | - ray stop --force - ALL_OFFLOAD=True VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 LR_WARMUP_STEPS=1 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with FP8 rollout - run: | - ray stop --force - export VLLM_USE_V1=1 - ROLLOUT_QUANTIZATION=fp8 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: clean up - run: | - rm -rf checkpoints - - cleanup: - runs-on: ubuntu-latest - needs: - [setup, e2e_ppo_trainer_megatron-deepseek, e2e_ppo_trainer_megatron-qwen3] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml b/.github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml deleted file mode 100644 index 83a1faf8832..00000000000 --- a/.github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml +++ /dev/null @@ -1,318 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_ppo_trainer_megatron_vllm_2 - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch. - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!verl/trainer/fsdp_sft_trainer.py" - # FSDP - - "!verl/workers/**/*dp_*.py" - - "!verl/utils/fsdp_utils.py" - - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" - - "!verl/model_merger/fsdp_model_merger.py" - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!docker/**" - # Docs - - "!**/*.md" - - "!docs/**" - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # FSDP - - "!verl/workers/**/*dp_*.py" - - "!verl/utils/fsdp_utils.py" - - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" - - "!verl/model_merger/fsdp_model_merger.py" - # Entrypoints - - ".github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml" - - "examples/data_preprocess/gsm8k.py" - - "examples/data_preprocess/geo3k.py" - - "tests/special_e2e/run_ppo_trainer_megatron.sh" - - "verl/trainer/main_ppo.py" - - "verl/trainer/config/ppo_megatron_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - e2e_ppo_trainer_megatron-moe-expert-parallel: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 60 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps --force-reinstall . - pip3 install git+https://github.com/NVIDIA-NeMo/Megatron-Bridge.git@83a7c11 --no-deps --no-build-isolation - pip3 install git+https://github.com/NVIDIA/Megatron-LM.git@5455f0a --no-deps --no-build-isolation - pip3 install "nvidia-modelopt[torch]>=0.37.0" transformers==4.57.1 - - name: Prepare GSM8K dataset - run: | - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron-Bridge (Qwen3-30B-A3B-Instruct-2507) - run: | - ray stop --force - ADV_ESTIMATOR=grpo USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json \ - PPO_MAX_TOKEN_LEN=1024 FWD_MAX_TOKEN_LEN=1024 \ - MAX_PROMPT_LENGTH=512 MAX_RESPONSE_LENGTH=512 \ - MODEL_ID=Qwen/Qwen3-30B-A3B-Instruct-2507 USE_MBRIDGE=True VANILLA_MBRIDGE=False VALUE_VANILLA_MBRIDGE=False \ - COMMON_PP=2 COMMON_VPP=null COMMON_CP=1 COMMON_TP=4 COMMON_EP=4 COMMON_ETP=1 INFER_TP=8 \ - USE_DIST_CKPT=True ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: Running GSM8K E2E training tests with 3D parallelism with FP8 rollout on 8 L20 GPUs with Megatron-Bridge (Qwen3-30B-A3B-Instruct-2507) - run: | - ray stop --force - ADV_ESTIMATOR=grpo USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json \ - PPO_MAX_TOKEN_LEN=1024 FWD_MAX_TOKEN_LEN=1024 \ - MAX_PROMPT_LENGTH=512 MAX_RESPONSE_LENGTH=512 \ - MODEL_ID=Qwen/Qwen3-30B-A3B-Instruct-2507 USE_MBRIDGE=True VANILLA_MBRIDGE=False VALUE_VANILLA_MBRIDGE=False \ - COMMON_PP=2 COMMON_VPP=null COMMON_CP=1 COMMON_TP=4 COMMON_EP=4 COMMON_ETP=1 INFER_TP=2 \ - USE_DIST_CKPT=True ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 ROLLOUT_QUANTIZATION=fp8 bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: clean up - run: | - rm -rf checkpoints - - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with Megatron-Bridge LoRA (Qwen3-30B-A3B-Instruct-2507) - run: | - ray stop --force - ADV_ESTIMATOR=grpo USE_DUMMY_MODEL=True DUMMY_MODEL_CONFIG_PATH=tests/special_e2e/ppo_trainer/expert_parallel/qwen2moe_minimal.json \ - PPO_MAX_TOKEN_LEN=1024 FWD_MAX_TOKEN_LEN=1024 \ - MAX_PROMPT_LENGTH=512 MAX_RESPONSE_LENGTH=512 LORA_RANK=8 CRITIC_LORA_RANK=8 \ - MODEL_ID=Qwen/Qwen3-30B-A3B-Instruct-2507 USE_MBRIDGE=True VANILLA_MBRIDGE=False VALUE_VANILLA_MBRIDGE=False \ - COMMON_PP=2 COMMON_VPP=null COMMON_CP=1 COMMON_TP=4 COMMON_EP=2 COMMON_ETP=1 INFER_TP=8 \ - USE_DIST_CKPT=False LORA_MERGE=True ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 bash tests/special_e2e/run_ppo_trainer_megatron.sh - - name: clean up - run: | - rm -rf checkpoints - - e2e_ppo_trainer_fsdp_vllm: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 60 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Prepare GSM8K dataset - run: | - ray stop --force - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - # Function RM - - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (FSDP_SIZE=8) - run: | - ray stop --force - VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp-size8" bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm after resuming - run: | - ray stop --force - RESUME_MODE=auto VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp-size8" bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Test merging FSDP checkpoints (Qwen Actor) - run: | - exp_name="qwen2.5-0.5b-function-reward-minimal-fsdp-size8" - python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface - - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (DDP_SIZE=2, FSDP_SIZE=4) - run: | - ray stop --force - VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True FSDP_SIZE=4 USE_KL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-ddp-size2-fsdp-size4" bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Test merging DDP+FSDP checkpoints (Qwen Actor) - run: | - exp_name="qwen2.5-0.5b-function-reward-minimal-ddp-size2-fsdp-size4" - python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface - - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (FSDP2) - run: | - ray stop --force - VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp2-size8" STRATEGY=fsdp2 bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Test merging FSDP2 checkpoints (Qwen Actor) - run: | - exp_name="qwen2.5-0.5b-function-reward-minimal-fsdp2-size8" - python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface - - name: Running GSM8K E2E without rmpad using function rm - run: | - ray stop --force - RM_PAD=False bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm (GRPO) - run: | - ray stop --force - CUSTOM_REWARD_FN=True ADV_ESTIMATOR=grpo USE_KL=True bash tests/special_e2e/ppo_trainer/run_function_reward.sh - # - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm (ReMax) - # run: | - # ray stop --force - # ADV_ESTIMATOR=remax USE_KL=True bash tests/special_e2e/ppo_trainer/run_function_reward.sh - # LoRA tests - - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm - run: | - ray stop --force - ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm and layered_summon - run: | - ray stop --force - ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors LAYERED_SUMMON=True TOTAL_TRAIN_STEPS=1 SAVE_FREQ=1 FSDP_SIZE=4 VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal" bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Test GRPO LoRA checkpoints merging function - run: | - export EXP_NAME="qwen2.5-0.5b-function-reward-minimal" - ls checkpoints/verl-test/${EXP_NAME}/global_step_1/actor - cat checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/huggingface/config.json - python3 -m verl.model_merger merge --backend fsdp --local_dir checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/ --target_dir checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/huggingface - - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm and layered_summon with fsdp2 - run: | - ray stop --force - ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors LAYERED_SUMMON=True STRATEGY=fsdp2 bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - e2e_ppo_trainer_fsdp-qwen2_5vl-3b: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 40 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - # Geo3k - - name: Prepare GEO3K dataset - run: | - python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ - - name: Running GEO3K VLM GRPO E2E training tests on 8 L20 GPUs with rmpad using function rm - run: | - ray stop --force - TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ - MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ - MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ - ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ - SP_SIZE=2 \ - bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - - name: Running GEO3K VLM PPO E2E training tests on 8 L20 GPUs with rmpad using function rm - run: | - ray stop --force - TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ - MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ - MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ - ADV_ESTIMATOR=gae RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ - SP_SIZE=2 \ - bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Running GEO3K VLM GRPO E2E lora training tests on 8 L20 GPUs with rmpad using function rm - run: | - ray stop --force - TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ - MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ - MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ - ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ - SP_SIZE=2 \ - LORA_RANK=32 LORA_EXCLUDE=".*visual.*" \ - bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - cleanup: - runs-on: ubuntu-latest - needs: - [ - setup, - e2e_ppo_trainer_megatron-moe-expert-parallel, - e2e_ppo_trainer_fsdp-qwen2_5vl-3b, - e2e_ppo_trainer_fsdp_vllm, - ] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/e2e_ppo_trainer_megatron_vllm_2_ascend.yml b/.github/workflows/e2e_ppo_trainer_megatron_vllm_2_ascend.yml deleted file mode 100644 index d0abdcc60e3..00000000000 --- a/.github/workflows/e2e_ppo_trainer_megatron_vllm_2_ascend.yml +++ /dev/null @@ -1,233 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_ppo_trainer_megatron_vllm_2_ascend - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch. - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!verl/trainer/fsdp_sft_trainer.py" - # FSDP - - "!verl/workers/**/*dp_*.py" - - "!verl/utils/fsdp_utils.py" - - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" - - "!verl/model_merger/fsdp_model_merger.py" - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!docker/**" - # Docs - - "!**/*.md" - - "!docs/**" - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # FSDP - - "!verl/workers/**/*dp_*.py" - - "!verl/utils/fsdp_utils.py" - - "!verl/utils/checkpoint/fsdp_checkpoint_manager.py" - - "!verl/model_merger/fsdp_model_merger.py" - # Entrypoints - - ".github/workflows/e2e_ppo_trainer_megatron_vllm_2_ascend.yml" - - "examples/data_preprocess/gsm8k.py" - - "examples/data_preprocess/geo3k.py" - - "tests/special_e2e/run_ppo_trainer_megatron.sh" - - "verl/trainer/main_ppo.py" - - "verl/trainer/config/ppo_megatron_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -jobs: - e2e_ppo_trainer_fsdp_vllm_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 90 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout verl-project/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e . - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Prepare GSM8K dataset - run: | - python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k - # Function RM - - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (DDP_SIZE=2, FSDP_SIZE=4) - run: | - ray stop --force - VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True FSDP_SIZE=4 USE_KL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-ddp-size2-fsdp-size4" bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Test merging DDP+FSDP checkpoints (Qwen Actor) - run: | - exp_name="qwen2.5-0.5b-function-reward-minimal-ddp-size2-fsdp-size4" - python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface - - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (FSDP2) - run: | - ray stop --force - VAL_BEFORE_TRAIN=True TEST_FREQ=1 SAVE_FREQ=1 SAVE_HF_MODEL=True VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal-fsdp2-size8" STRATEGY=fsdp2 bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Test merging FSDP2 checkpoints (Qwen Actor) - run: | - exp_name="qwen2.5-0.5b-function-reward-minimal-fsdp2-size8" - python -m verl.model_merger test --backend fsdp --local_dir checkpoints/verl-test/${exp_name}/global_step_1/actor --test_hf_dir checkpoints/verl-test/${exp_name}/global_step_1/actor/huggingface - - name: Running GSM8K E2E without rmpad using function rm - run: | - ray stop --force - RM_PAD=False bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm (GRPO) - run: | - ray stop --force - CUSTOM_REWARD_FN=True ADV_ESTIMATOR=grpo USE_KL=True bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm and layered_summon - run: | - ray stop --force - ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors LAYERED_SUMMON=True TOTAL_TRAIN_STEPS=1 SAVE_FREQ=1 FSDP_SIZE=4 VERL_EXP_NAME="qwen2.5-0.5b-function-reward-minimal" bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Test GRPO LoRA checkpoints merging function - run: | - export EXP_NAME="qwen2.5-0.5b-function-reward-minimal" - ls checkpoints/verl-test/${EXP_NAME}/global_step_1/actor - cat checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/huggingface/config.json - python3 -m verl.model_merger merge --backend fsdp --local_dir checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/ --target_dir checkpoints/verl-test/${EXP_NAME}/global_step_1/actor/huggingface - - name: Running GSM8K E2E training tests on 8 L20 GPUs with grpo lora using function rm with use_shm and layered_summon with fsdp2 - run: | - ray stop --force - ADV_ESTIMATOR=grpo USE_SHM=True LORA_RANK=32 LOAD_FORMAT=safetensors LAYERED_SUMMON=True STRATEGY=fsdp2 bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - e2e_ppo_trainer_fsdp-qwen2_5vl-3b_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 60 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout verl-project/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e . - pip install trl==0.26.0 - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - # Geo3k - - name: Prepare GEO3K dataset - run: | - python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k - - name: Running GEO3K VLM GRPO E2E training tests on 8 L20 GPUs with rmpad using function rm - run: | - ray stop --force - TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ - MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ - MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ - ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ - SP_SIZE=2 \ - bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Running GEO3K VLM PPO E2E training tests on 8 L20 GPUs with rmpad using function rm - run: | - ray stop --force - TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ - MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ - MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ - ADV_ESTIMATOR=gae RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ - SP_SIZE=2 \ - bash tests/special_e2e/ppo_trainer/run_function_reward.sh - - name: Running GEO3K VLM GRPO E2E lora training tests on 8 L20 GPUs with rmpad using function rm - run: | - ray stop --force - TRAIN_FILES=$HOME/data/geo3k/train.parquet VAL_FILES=$HOME/data/geo3k/test.parquet \ - MAX_PROMPT_LEN=1536 MAX_RESPONSE_LEN=1536 \ - MODEL_ID=Qwen/Qwen2.5-VL-3B-Instruct \ - ADV_ESTIMATOR=grpo RM_PAD=True USE_KL=True ENABLE_CHUNKED_PREFILL=False \ - SP_SIZE=2 \ - LORA_RANK=32 LORA_EXCLUDE=".*visual.*" \ - bash tests/special_e2e/ppo_trainer/run_function_reward.sh diff --git a/.github/workflows/e2e_ppo_trainer_veomni_vllm.yml b/.github/workflows/e2e_ppo_trainer_veomni_vllm.yml deleted file mode 100644 index 0accafd58e8..00000000000 --- a/.github/workflows/e2e_ppo_trainer_veomni_vllm.yml +++ /dev/null @@ -1,153 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_ppo_trainer_veomni_vllm - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch. - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!verl/trainer/fsdp_sft_trainer.py" - # Megatron - - "!verl/workers/**/megatron_*.py" - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!docker/**" - # Docs - - "!**/*.md" - - "!docs/**" - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # Megatron - - "!verl/workers/**/megatron_*.py" - # Entrypoints - - ".github/workflows/e2e_ppo_trainer_veomni_vllm.yml" - - "examples/data_preprocess/gsm8k.py" - - "examples/data_preprocess/geo3k.py" - - "tests/special_e2e/run_ppo_trainer_veomni.sh" - - "verl/trainer/main_ppo.py" - - "verl/trainer/config/ppo_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - e2e_ppo_trainer_veomni_vllm: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 60 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install git+https://github.com/ByteDance-Seed/VeOmni.git@v0.1.4 - - name: Prepare GSM8K dataset - run: | - ray stop --force - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Prepare GEO3K dataset - run: | - ray stop --force - python3 examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/models/hf_data/hiyouga/geometry3k/ - - name: Running GSM8K E2E training tests on 8 L20 GPUs with veomni engine (FSDP_SIZE=4, USP=2) - run: | - ray stop --force - FSDP_SIZE=4 SP_SIZE=2 bash tests/special_e2e/run_ppo_trainer_veomni.sh - - name: Running GEO3K E2E training tests on 8 L20 GPUs with veomni engine (FSDP_SIZE=8, USP=1) - run: | - ray stop --force - MODEL_ID=Qwen/Qwen3-VL-2B-Instruct TRAIN_FILES=${HOME}/data/geo3k/train.parquet VAL_FILES=${HOME}/data/gsm8k/test.parquet FSDP_SIZE=8 SP_SIZE=1 bash tests/special_e2e/run_ppo_trainer_veomni.sh - - cleanup: - runs-on: ubuntu-latest - needs: - [ - setup, - e2e_ppo_trainer_veomni_vllm, - ] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/e2e_sft_llm.yml b/.github/workflows/e2e_sft_llm.yml deleted file mode 100644 index 435a0a626db..00000000000 --- a/.github/workflows/e2e_sft_llm.yml +++ /dev/null @@ -1,153 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_sft_llm - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - - # Megatron - - "!verl/workers/**/megatron_*.py" - # Entrypoints - - ".github/workflows/e2e_sft_llm.yml" - - "examples/data_preprocess/gsm8k.py" - - "tests/special_e2e/sft" - - "verl/trainer/fsdp_sft_trainer.py" - - "verl/trainer/config/sft_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - e2e_sft_llm: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 30 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install peft - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install git+https://github.com/ByteDance-Seed/VeOmni.git@v0.1.4 - - name: Prepare gsm8k dataset - run: | - ray stop --force - python3 examples/data_preprocess/gsm8k_multiturn_sft.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm - run: | - ray stop --force - bash tests/special_e2e/sft/run_sft.sh - - name: Running GSM8K E2E training tests on 8 L20 GPUs w/o rmpad using function rm - run: | - ray stop --force - RM_PAD=False bash tests/special_e2e/sft/run_sft.sh - - name: Running GSM8K E2E training tests on 8 L20 GPUs with sequence parallism - run: | - ray stop --force - SP_SIZE=2 bash tests/special_e2e/sft/run_sft.sh - - name: Running GSM8K E2E training tests on 8 L20 GPUs with sequence parallism and liger - run: | - ray stop --force - SP_SIZE=2 LIGER=True bash tests/special_e2e/sft/run_sft.sh - - name: Running GSM8K E2E training tests with LoRA - run: | - ray stop --force - LORA_RANK=32 bash tests/special_e2e/sft/run_sft.sh - - name: Run GSM8K E2E training and resume tests resuming from the checkpoint manager - run: | - ray stop --force - LORA_RANK=32 RESUME_MODE=auto TOTAL_TRAIN_STEP=2 bash tests/special_e2e/sft/run_sft.sh - # TODO: multiturn - - name: Running GSM8K E2E training tests with multiturn and various configs and compare results - run: | - bash tests/special_e2e/sft/test_sft_engine_all.sh - - cleanup: - runs-on: ubuntu-latest - needs: [setup, e2e_sft_llm] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/e2e_sft_llm_ascend.yml b/.github/workflows/e2e_sft_llm_ascend.yml deleted file mode 100644 index 3919da747a9..00000000000 --- a/.github/workflows/e2e_sft_llm_ascend.yml +++ /dev/null @@ -1,160 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_sft_llm_ascend - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - - # Megatron - - "!verl/workers/**/megatron_*.py" - # Entrypoints - - ".github/workflows/e2e_sft_llm_ascend.yml" - - "examples/data_preprocess/gsm8k.py" - - "tests/special_e2e/sft" - - "verl/trainer/fsdp_sft_trainer.py" - - "verl/trainer/config/sft_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -jobs: - e2e_sft_llm_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 90 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout verl-project/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install -e . - pip install git+https://github.com/ByteDance-Seed/VeOmni.git@v0.1.4 - pip install pandas==2.3.3 - pip uninstall -y mbridge - pip install git+https://github.com/ISEEKYAN/mbridge.git@89eb10 - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Prepare gsm8k dataset - run: | - python3 examples/data_preprocess/gsm8k_multiturn_sft.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k - - name: Running GSM8K E2E training tests on 8 NPUs with rmpad using function rm - run: | - ray stop --force - bash tests/special_e2e/sft/run_sft.sh - - name: Running GSM8K E2E training tests on 8 NPUs w/o rmpad using function rm - run: | - ray stop --force - RM_PAD=False bash tests/special_e2e/sft/run_sft.sh - - name: Running GSM8K E2E training tests on 8 NPUs with sequence parallism - run: | - ray stop --force - SP_SIZE=2 bash tests/special_e2e/sft/run_sft.sh - - name: Running GSM8K E2E training tests with LoRA - run: | - ray stop --force - LORA_RANK=32 bash tests/special_e2e/sft/run_sft.sh - - name: Run GSM8K E2E training and resume tests resuming from the checkpoint manager - run: | - ray stop --force - LORA_RANK=32 RESUME_MODE=auto TOTAL_TRAIN_STEP=2 bash tests/special_e2e/sft/run_sft.sh - - name: Running GSM8K E2E training tests with multiturn and various configs and compare results - run: | - ray stop --force - rm -rf ~/verl/test/log - mkdir -p ~/verl/test/log - export VERL_FILE_LOGGER_ROOT=~/verl/test/log - # test with single gpu as golden - echo "run with single gpu as golden" - BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=1 NUM_GPUS=1 FSDP_STRATEGY=fsdp VERL_FILE_LOGGER_PATH=~/verl/test/log/golden.jsonl bash tests/special_e2e/sft/run_sft_engine.sh - # test with fsdp 1 - echo "run with sp2 fsdp_size2 num_gpus8 fsdp_strategy fsdp pad_mode no_padding" - BACKEND=fsdp SP_SIZE=2 FSDP_SIZE=2 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=no_padding bash tests/special_e2e/sft/run_sft_engine.sh - # test with fsdp 1 use_remove_padding and pad_mode no_padding - echo "run with sp4 fsdp_size4 num_gpus8 fsdp_strategy fsdp pad_mode no_padding use_remove_padding False" - BACKEND=fsdp SP_SIZE=1 FSDP_SIZE=-1 NUM_GPUS=8 FSDP_STRATEGY=fsdp PAD_MODE=no_padding USE_REMOVE_PADDING=False bash tests/special_e2e/sft/run_sft_engine.sh - # test with fsdp 2 - echo "run with sp2 fsdp_size2 num_gpus8 fsdp_strategy fsdp2" - BACKEND=fsdp SP_SIZE=2 FSDP_SIZE=2 NUM_GPUS=8 FSDP_STRATEGY=fsdp2 bash tests/special_e2e/sft/run_sft_engine.sh - # test with veomni - echo "run with sp2 fsdp_size4 num_gpus8 fsdp_strategy fsdp2" - BACKEND=veomni SP_SIZE=2 FSDP_SIZE=4 NUM_GPUS=8 FSDP_STRATEGY=fsdp2 bash tests/special_e2e/sft/run_sft_engine.sh - # test with megatron - echo "run with tp2 pp2 vpp2 cp2 num_gpus8" - BACKEND=megatron TP_SIZE=2 PP_SIZE=2 VPP_SIZE=NULL CP_SIZE=2 NUM_GPUS=8 bash tests/special_e2e/sft/run_sft_engine.sh - # test with cp in ray - echo "run with tp2 pp2 vpp2 cp2 num_gpus8 mode=ray" - BACKEND=megatron TP_SIZE=2 PP_SIZE=2 VPP_SIZE=NULL CP_SIZE=2 NUM_GPUS=8 mode=ray bash tests/special_e2e/sft/run_sft_engine.sh - rm -rf ~/verl/test/log diff --git a/.github/workflows/e2e_sft_vlm.yml b/.github/workflows/e2e_sft_vlm.yml deleted file mode 100644 index 93d02c83c8c..00000000000 --- a/.github/workflows/e2e_sft_vlm.yml +++ /dev/null @@ -1,128 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: e2e_sft_vlm - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - - # Megatron - - "!verl/workers/**/megatron_*.py" - # Entrypoints - - ".github/workflows/e2e_sft_vlm.yml" - - "examples/data_preprocess/gsm8k.py" - - "tests/special_e2e/sft" - - "verl/trainer/fsdp_sft_trainer.py" - - "verl/trainer/config/sft_trainer.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - e2e_sft_vlm: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 30 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install peft - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install git+https://github.com/ByteDance-Seed/VeOmni.git@v0.1.4 - - name: Prepare pokemon-gpt4o-captions dataset - run: | - ray stop --force - python3 examples/data_preprocess/pokemon.py --local_dataset_path ${HOME}/models/hf_data/pokemon-gpt4o-captions - - name: Running Pokemon E2E training tests with multiturn and various configs and compare results - run: | - MODEL_ID=Qwen/Qwen3-VL-2B-Instruct DATASET_DIR=~/data/pokemon-gpt4o-captions VPP_SIZE=null bash tests/special_e2e/sft/test_sft_engine_all.sh - - cleanup: - runs-on: ubuntu-latest - needs: [setup, e2e_sft_vlm] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/gpu_unit_tests.yml b/.github/workflows/gpu_unit_tests.yml deleted file mode 100644 index 6c16b950afd..00000000000 --- a/.github/workflows/gpu_unit_tests.yml +++ /dev/null @@ -1,137 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: GPU unit tests - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.4.x - paths: - - "**/*.py" - - .github/workflows/gpu_unit_tests.yml - pull_request: - branches: - - main - - v0.4.x - paths: - # The order that you define paths patterns matters: - # A matching negative pattern (prefixed with !) after a positive match will exclude the path. - # A matching positive pattern after a negative match will include the path again. - - "**/*.py" - # Other entrypoints - - "!examples/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # Entrypoints - - .github/workflows/gpu_unit_tests.yml - - "tests/**test_*.py" - # Ignore CPU tests - - "!tests/*_on_cpu.py" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - gpu_unit_tests: - if: github.repository_owner == 'verl-project' - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 60 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1" - HF_HUB_ENABLE_HF_TRANSFER: 1 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install hf_transfer - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install cupy-cuda12x==13.6.0 pytest-asyncio - pip3 install --ignore-installed blinker - pip3 install --ignore-installed mlflow "numpy<2.0" - - name: Run all GPU unit tests - run: | - pytest -s -x --ignore-glob="*on_npu.py" --ignore-glob="*test_special_*.py" --ignore-glob='*on_cpu.py' --ignore-glob="*test_vllm*" --ignore-glob="*_sglang*" --ignore-glob="*_hf_rollout*" --ignore-glob="tests/models/" --ignore-glob='tests/special*' --ignore-glob="tests/experimental" --ignore-glob="tests/workers/reward_model" --ignore-glob="*test_shared_memory*" --ignore-glob="tests/workers/rollout/rollout_trtllm" --ignore-glob="*test_bucketed_weight_transfer*" tests/ - - name: Testing LinearCrossEntropyTP Correctness, Computation Time and Memory Consumption - run: | - LOW_MEMORY=True torchrun --standalone --nnodes=1 --nproc-per-node=8 tests/utils/test_special_linear_cross_entropy_tp.py - - name: Testing FSDP2 actor functionality - run: | - torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/workers/actor/test_special_dp_actor.py - - name: Testing FSDP2 critic functionality - run: | - torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/workers/critic/test_special_dp_critic.py - - cleanup: - runs-on: ubuntu-latest - needs: [setup, gpu_unit_tests] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/model.yml b/.github/workflows/model.yml deleted file mode 100644 index 5522ba71466..00000000000 --- a/.github/workflows/model.yml +++ /dev/null @@ -1,184 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. -# name: Check PR Title - -name: model - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "verl/**/*.py" - # Entrypoints - - ".github/workflows/model.yml" - - "tests/special_distributed/test_fsdp_ckpt.py" - - "tests/special_distributed/test_tensor_dict.py" - - "tests/models/**" - - "tests/special_distributed/run_all.sh" - -# Declare permissions just read content. -permissions: - contents: read - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - model_rmpad: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 20 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository and upgrade to latest transformers(4.54.0)/flash_attn, transformers 4.55.0 has strange behavior with model backward - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install --upgrade "transformers<5.0.0" - - name: Running rmpad model tests on 8 L20 GPUs + flash_attn 2.5.8 - run: | - pytest -s tests/models/test_transformer.py - - name: Running rmpad model tests on 8 L20 GPUs + latest flash_attn - run: | - pytest -s tests/models/test_transformer.py - - name: Running FSDP rmpad model tests on 8 L20 GPUs + latest flash_attn - run: | - STRATEGY=fsdp torchrun --nproc_per_node=8 tests/special_distributed/test_fsdp_ckpt.py - - name: Running transformers ulysses tests on 8 L20 GPUs + latest transformers - run: | - torchrun --nproc_per_node=8 -m pytest tests/models/test_transformers_ulysses.py - - name: Running transformers ulysses tests on 8 L20 GPUs + transformers 4.54.1 - run: | - pip3 install transformers==4.54.1 - torchrun --nproc_per_node=8 -m pytest tests/models/test_transformers_ulysses.py - - name: Run distributed test - run: | - bash tests/special_distributed/run_all.sh - - # TODO: Move this back to model_rmpad once FSDP2 is stable. - # NOTE: List as an independent job to make rerun easier. - model_rmpad_fsdp2_unstable: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 20 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository and upgrade to latest transformers/flash_attn - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Running FSDP2 rmpad model tests on 8 L20 GPUs + latest flash_attn - run: | - STRATEGY=fsdp2 torchrun --nproc_per_node=8 tests/special_distributed/test_fsdp_ckpt.py - - model_engine: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 20 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Download model config files - run: | - hf download Qwen/Qwen2.5-0.5B-Instruct --local-dir $HOME/models/Qwen/Qwen2.5-0.5B-Instruct - - - name: Running mcore engine tests on 8 L20 GPUs - run: | - ray stop --force - pytest -s -x tests/models/test_engine.py - - cleanup: - runs-on: ubuntu-latest - needs: [setup, model_rmpad, model_rmpad_fsdp2_unstable, model_engine] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/model_ascend.yml b/.github/workflows/model_ascend.yml deleted file mode 100644 index a5ab7620ee3..00000000000 --- a/.github/workflows/model_ascend.yml +++ /dev/null @@ -1,137 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. -# name: Check PR Title - -name: model_ascend - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "verl/**/*.py" - # Entrypoints - - ".github/workflows/model_ascend.yml" - - "tests/special_distributed/test_fsdp_ckpt.py" - - "tests/special_distributed/test_tensor_dict.py" - - "tests/models/**" - - "tests/special_distributed/run_all.sh" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -permissions: - contents: read - -jobs: - model_rmpad_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 60 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout verl-project/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e .[test] - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Running rmpad model tests on 8 NPUs - run: | - pytest -s tests/models/test_transformer.py - - name: Running FSDP rmpad model tests on 8 NPUs - run: | - STRATEGY=fsdp torchrun --nproc_per_node=8 tests/special_distributed/test_fsdp_ckpt.py - - name: Running transformers ulysses tests on 8 NPUs - run: | - torchrun --nproc_per_node=8 -m pytest tests/models/test_transformers_ulysses.py - - name: Run distributed test - run: | - bash tests/special_distributed/run_all.sh - - # TODO: Move this back to model_rmpad once FSDP2 is stable. - # NOTE: List as an independent job to make rerun easier. - model_rmpad_fsdp2_unstable_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 60 - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e .[test] - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Running FSDP2 rmpad model tests on 8 NPUs - run: | - STRATEGY=fsdp2 torchrun --nproc_per_node=8 tests/special_distributed/test_fsdp_ckpt.py diff --git a/.github/workflows/nightly_ascend.yml b/.github/workflows/nightly_ascend.yml deleted file mode 100644 index c74ea4ba7d9..00000000000 --- a/.github/workflows/nightly_ascend.yml +++ /dev/null @@ -1,174 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: nightly_ci_ascend - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - # For push, for now only anti-patterns are specified so it is more conservative - # and achieves higher coverage. - schedule: - - cron: "0 17 * * *" - -# Declare permissions just read content. -permissions: - contents: read - -jobs: - # Test ppo qwen3-8b fsdp+vllm - nightlyCI_ppo-qwen3-8b-fsdp-vllm_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 180 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout verl-project/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e . - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Prepare GSM8K dataset - run: | - python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k - - name: Running nightlyCI_ppo-qwen3-8b-fsdp-vllm_ascend - run: | - ray stop --force - bash tests/special_npu/nightly_ci_ascend/run_ppo_qwen3-8b_fsdp_npu.sh - - # Test grpo qwen25-7b-Instruct fsdp+vllm - nightlyCI_grpo-qwen25-7b-Instruct-fsdp-vllm_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 180 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout verl-project/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e . - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Prepare GSM8K dataset - run: | - python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k - - name: Running nightlyCI_grpo-qwen25-7b-Instruct-fsdp-vllm_ascend - run: | - ray stop --force - bash tests/special_npu/nightly_ci_ascend/run_grpo_qwen25-7b-instruct_fsdp_npu.sh - - # Test grpo qwen25-vl-3b-Instruct fsdp+vllm - nightlyCI_grpo-qwen25-vl-3b-Instruct-fsdp-vllm_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 180 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout verl-project/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e . - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Preprocess geo3k dataset - run: | - python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k - - name: Running nightlyCI_grpo-qwen25-vl-3b-Instruct-fsdp-vllm_ascend - run: | - ray stop --force - bash tests/special_npu/nightly_ci_ascend/run_grpo_qwen25-vl-3b-instruct_fsdp_npu.sh diff --git a/.github/workflows/npu_unit_tests.yml b/.github/workflows/npu_unit_tests.yml deleted file mode 100644 index 7f678409da0..00000000000 --- a/.github/workflows/npu_unit_tests.yml +++ /dev/null @@ -1,126 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - `npu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix on ascend device. -# - Since cpu/gpu/npu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: NPU unit tests - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - - .github/workflows/npu_unit_tests.yml - pull_request: - branches: - - main - paths: - # The order that you define paths patterns matters: - # A matching negative pattern (prefixed with !) after a positive match will exclude the path. - # A matching positive pattern after a negative match will include the path again. - - "**/*.py" - # Other entrypoints - - "!examples/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - - "!recipe/**" - # Entrypoints - - .github/workflows/npu_unit_tests.yml - - "tests/**test_*.py" - # Ignore CPU tests - - "!tests/*_on_cpu.py" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -jobs: - npu_unit_tests: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 60 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout volcengine/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e .[test] - pip install mlflow pytest-asyncio - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Run all NPU unit tests - run: | - pytest -s -x --ignore-glob="*test_special_*.py" --ignore-glob="*on_cpu.py" --ignore-glob="*test_vllm*" --ignore-glob="*_sglang*" --ignore-glob="*_hf_rollout*" --ignore-glob="tests/models/" --ignore-glob="tests/special*" --ignore-glob="tests/experimental" --ignore-glob="tests/workers/reward_model" --ignore-glob="*test_rvdz*" --ignore-glob="*test_ray_collectives*" --ignore-glob="*test_nvtx_profile*" --ignore-glob="tests/checkpoint_engine" --ignore-glob="*test_shared_memory*" --ignore-glob="tests/workers/rollout/rollout_trtllm" --ignore-glob="*test_fsdp_lora_merge*" --ignore-glob="*test_activation_offload*" --ignore-glob="*test_normalize_peft_param_name.py*" tests/ - - name: Testing activation offload - run: | - pytest -s -x tests/utils/test_activation_offload.py - - name: Testing normalize peft param name - run: | - pytest -s -x tests/utils/test_normalize_peft_param_name.py - - name: Testing FSDP2 actor functionality - run: | - torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/workers/actor/test_special_dp_actor.py - - name: Testing FSDP2 critic functionality - run: | - torchrun --standalone --nnodes=1 --nproc-per-node=2 tests/workers/critic/test_special_dp_critic.py - - name: Running NPU profiling unit tests - run: | - pytest -s -x tests/utils/test_special_mstx_profile.py diff --git a/.github/workflows/precommit-autofix.yml b/.github/workflows/precommit-autofix.yml deleted file mode 100644 index d235da90cd2..00000000000 --- a/.github/workflows/precommit-autofix.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: scheduled pre-commit autofix - -on: - schedule: - # Every hour - - cron: "0 * * * *" - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - -jobs: - precommit: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install pre-commit - run: | - python -m pip install --upgrade pip - pip install pre-commit hydra-core - - - name: Run pre-commit - run: | - pre-commit run --all-files || true - - - name: Create or update PR - uses: peter-evans/create-pull-request@v6 - with: - branch: bot/precommit-autofix - delete-branch: true - title: "[ci] chore: scheduled pre-commit autofix" - commit-message: "chore: auto-fix pre-commit issues" - body: | - This PR was created automatically by a scheduled GitHub Action. - - - Runs `pre-commit run --all-files` - - Triggered hourly - labels: | - automated - pre-commit diff --git a/.github/workflows/reward_model_sglang.yml b/.github/workflows/reward_model_sglang.yml deleted file mode 100644 index c9a4e9804a0..00000000000 --- a/.github/workflows/reward_model_sglang.yml +++ /dev/null @@ -1,134 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. -# name: Check PR Title - -name: reward_model_sglang - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "verl/**/*.py" - # Entrypoints - - ".github/workflows/reward_model_sglang.yml" - - "tests/experimental/reward_loop/**" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - reward_model_sglang: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 30 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK: "True" - NCCL_SHM_DISABLE: "1" - NCCL_P2P_DISABLE: "1" - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install sglang-router==0.2.2 - - name: Prepare gsm8k dataset - run: | - ray stop --force - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_dir ${HOME}/data/gsm8k - - name: Running sglang generative reward model tests on 8 L20 GPUs - run: | - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY - ROLLOUT_NAME=sglang pytest -s -x tests/experimental/reward_loop/test_reward_model_genrm.py - - name: Running sglang discriminative reward model tests on 8 L20 GPUs - run: | - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY - ROLLOUT_NAME=sglang pytest -s -x tests/experimental/reward_loop/test_reward_model_disrm.py - - name: Running sglang agent loop with reward manager tests on 8 L20 GPUs - run: | - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY - ROLLOUT_NAME=sglang pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_standalone.py - - name: Running sglang agent loop with reward model colocate tests on 8 L20 GPUs - run: | - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY - ROLLOUT_NAME=sglang pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_colocate.py - - cleanup: - runs-on: ubuntu-latest - needs: [setup, reward_model_sglang] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/reward_model_vllm.yml b/.github/workflows/reward_model_vllm.yml deleted file mode 100644 index aebde06984f..00000000000 --- a/.github/workflows/reward_model_vllm.yml +++ /dev/null @@ -1,134 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. -# name: Check PR Title - -name: reward_model_vllm - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "verl/**/*.py" - # Entrypoints - - ".github/workflows/reward_model_vllm.yml" - - "tests/experimental/reward_loop/**" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - reward_model_vllm: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 30 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK: "True" - NCCL_SHM_DISABLE: "1" - NCCL_P2P_DISABLE: "1" - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Prepare gsm8k dataset - run: | - ray stop --force - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_dir ${HOME}/data/gsm8k - - name: Running vllm generative reward model tests on 8 L20 GPUs - run: | - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY - ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_reward_model_genrm.py - - name: Running vllm discriminative reward model tests on 8 L20 GPUs - run: | - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY - ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_reward_model_disrm.py - - - name: Running vllm agent loop with reward manager tests on 8 L20 GPUs - run: | - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY - ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_standalone.py - - name: Running vllm agent loop with reward model colocate tests on 8 L20 GPUs - run: | - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY - ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_colocate.py - - cleanup: - runs-on: ubuntu-latest - needs: [setup, reward_model_vllm] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/reward_model_vllm_ascend.yml b/.github/workflows/reward_model_vllm_ascend.yml deleted file mode 100644 index b57aa97c73b..00000000000 --- a/.github/workflows/reward_model_vllm_ascend.yml +++ /dev/null @@ -1,113 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. -# name: Check PR Title - -name: reward_model_vllm_ascend - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "verl/**/*.py" - # Entrypoints - - ".github/workflows/reward_model_vllm_ascend.yml" - - "tests/experimental/reward_loop/**" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -jobs: - reward_model_vllm_ascend: - if: github.repository_owner == 'verl-project' - runs-on: linux-aarch64-a2b3-8 - timeout-minutes: 60 # Increase this timeout value as needed - container: - image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:verl-8.5.0-910b-ubuntu22.04-py3.11-latest - options: >- - --shm-size 16g - env: - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - name: Check npu and CANN info - run: | - cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info - npu-smi info - - name: Check initial pip list from image - run: | - pip list - - name: Checkout verl-project/verl repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true - - name: Install the current repository - run: | - pip install -r requirements-npu.txt - pip install --no-deps -e .[test] - - name: Check final pip list - run: | - pip list - - name: Prepare weights - run: | - ln -s /root/.cache/models ~/models - - name: Prepare gsm8k dataset - run: | - ray stop --force - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k --local_dir ${HOME}/data/gsm8k - - name: Running vllm generative reward model tests on 8 NPUs - run: | - ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_reward_model_genrm.py - - name: Running vllm discriminative reward model tests on 8 NPUs - run: | - ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_reward_model_disrm.py - - name: Running vllm agent loop with reward manager tests on 8 NPUs - run: | - ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_standalone.py - - name: Running vllm agent loop with reward model colocate tests on 8 NPUs - run: | - export HCCL_HOST_SOCKET_PORT_RANGE=auto - export HCCL_NPU_SOCKET_PORT_RANGE=auto - ROLLOUT_NAME=vllm pytest -s -x tests/experimental/reward_loop/test_agent_reward_loop_colocate.py \ No newline at end of file diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml deleted file mode 100644 index ac7532d2f04..00000000000 --- a/.github/workflows/sanity.yml +++ /dev/null @@ -1,108 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. -# name: Check PR Title - -name: sanity - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - - .github/workflows/sanity.yml - - "tests/special_sanity/**" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -jobs: - sanity: - runs-on: ubuntu-latest - timeout-minutes: 5 # Increase this timeout value as needed - strategy: - matrix: - python-version: ["3.10"] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: ${{ matrix.python-version }} - - name: Install the current repository - run: | - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu - pip3 install -r requirements.txt - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Run sanity test - run: | - pytest -s -x tests/special_sanity - - name: Run license test - run: | - python3 tests/special_sanity/check_license.py --directories . - - name: Assert naming convention - run: | - if grep -rIn --exclude-dir=.git --exclude-dir=.github --exclude-dir=venv --exclude-dir=__pycache__ 'veRL' .; then - echo "Please use verl instead of veRL in the codebase" - exit 1 - fi - - name: Assert SGLang naming convention - run: | - if grep -rIn --exclude-dir=.git --exclude-dir=.github --exclude-dir=venv --exclude-dir=__pycache__ --exclude=ascend_sglang_best_practices.rst -E 'Sglang|sgLang|sglAng|sglaNg|sglanG' .; then - echo "Please use SGLang or sglang as the formal name of SGLang rollout engine" - exit 1 - fi - - name: Validate test folder structure - run: python3 tests/special_sanity/validate_structure.py - - name: Assert documentation requirement for functions - run: python3 tests/special_sanity/validate_imported_docs.py - - name: Assert device api usage in verl/verl - run: python3 tests/special_sanity/check_device_api_usage.py --directory ./verl - - name: Assert documentation time info - run: python3 tests/special_sanity/check_docs_time_info.py - - name: Check docstrings for specified files - run: python3 tests/special_sanity/check_docstrings.py - - name: Check DataProto for specified folders - run: python3 tests/special_sanity/check_dataproto_usage.py -d ./verl/workers/engine diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml deleted file mode 100644 index 176d15ae2bd..00000000000 --- a/.github/workflows/scorecard.yml +++ /dev/null @@ -1,66 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. They are provided -# by a third-party and are governed by separate terms of service, privacy -# policy, and support documentation. - -name: Scorecard supply-chain security -on: - # For Branch-Protection check. Only the default branch is supported. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection - branch_protection_rule: - # To guarantee Maintained check is occasionally updated. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained - schedule: - - cron: "27 7 * * 1" - push: - branches: - - main - - v0.* - -# Declare default permissions as read only. -permissions: read-all - -jobs: - analysis: - name: Scorecard analysis - runs-on: ubuntu-latest - permissions: - # Needed to upload the results to code-scanning dashboard. - security-events: write - # Needed to publish results and get a badge (see publish_results below). - id-token: write - # Uncomment the permissions below if installing in a private repository. - # contents: read - # actions: read - - steps: - - name: "Checkout code" - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - persist-credentials: false - - - name: "Run analysis" - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 - with: - results_file: results.sarif - results_format: sarif - # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: - # - you want to enable the Branch-Protection check on a *public* repository, or - # - you are installing Scorecard on a *private* repository - # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. - # repo_token: ${{ secrets.SCORECARD_TOKEN }} - - # Public repositories: - # - Publish results to OpenSSF REST API for easy access by consumers - # - Allows the repository to include the Scorecard badge. - # - See https://github.com/ossf/scorecard-action#publishing-results. - # For private repositories: - # - `publish_results` will always be set to `false`, regardless - # of the value entered here. - publish_results: true - - # Upload the results to GitHub's code scanning dashboard (optional). - # Commenting out will disable upload of results to your repo's Code Scanning dashboard - - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 #v3.28.9 - with: - sarif_file: results.sarif diff --git a/.github/workflows/secrets_scan.yml b/.github/workflows/secrets_scan.yml deleted file mode 100644 index 298ed16c668..00000000000 --- a/.github/workflows/secrets_scan.yml +++ /dev/null @@ -1,22 +0,0 @@ -on: - push: - branches: - - main - - v0.* - pull_request: - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - name: Secret Scanning - uses: trufflesecurity/trufflehog@7dc056a193116ba8d82154bf0549381c8fb8545c # v3.88.14 - with: - extra_args: --results=verified,unknown diff --git a/.github/workflows/sgl.yml b/.github/workflows/sgl.yml deleted file mode 100644 index bc0c0bb7f4a..00000000000 --- a/.github/workflows/sgl.yml +++ /dev/null @@ -1,165 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: sgl - -on: - # workflow_dispatch: # Manual - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - paths: - - "**/*.py" - - .github/workflows/sgl.yml - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" # FSDP - - "!verl/workers/**/*dp_*.py" - # Megatron - - "!verl/workers/**/megatron_*.py" - # vLLM - - "!**/*vllm*" - - # Entrypoints - - ".github/workflows/sgl.yml" - - "tests/rollout/*sglang*" - - "tests/rollout/async_rollout_utils.py" - - "tests/workers/rollout/*interaction*" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:sgl059.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - sgl: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 35 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: 1 - SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK: "True" - NCCL_SHM_DISABLE: "1" - NCCL_P2P_DISABLE: "1" - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install cupy-cuda12x==13.6.0 pytest-asyncio - pip3 install hf_transfer fastmcp pytest-asyncio - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Prepare gsm8k dataset - run: | - ray stop --force - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Test the latest SGLang Rollout async with agent loop - run: | - ROLLOUT_NAME=sglang pytest -svvv tests/experimental/agent_loop - - sgl_checkpoint_engine: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 35 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: 1 - SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK: "True" - NCCL_SHM_DISABLE: "1" - NCCL_P2P_DISABLE: "1" - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install cupy-cuda12x==13.6.0 pytest-asyncio - pip3 install hf_transfer fastmcp pytest-asyncio - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - - name: Test SGLang ServerAdapter with Checkpoint Engine (NCCL) - run: | - ROLLOUT_NAME=sglang pytest -svvv tests/checkpoint_engine/test_special_server_adapter.py - - cleanup: - runs-on: ubuntu-latest - needs: [setup, sgl, sgl_checkpoint_engine] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows/type-coverage-check.yml b/.github/workflows/type-coverage-check.yml deleted file mode 100644 index 268f0c672f0..00000000000 --- a/.github/workflows/type-coverage-check.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Type Annotation and Docstring Coverage - -on: - pull_request: - paths: - - '**/*.py' - - '.github/workflows/type-coverage-check.yml' - -jobs: - type-coverage-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # 🚨 Important: fetch full history so `origin/main` is available - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Install dependencies - run: | - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu - pip3 install -r requirements.txt - pip3 install --no-deps -e . - - name: Run type annotation coverage check - run: | - python3 tests/special_sanity/type_coverage_check.py - - name: Run docstring coverage check - run: | - python3 tests/special_sanity/check_api_docs.py verl diff --git a/.github/workflows/vllm.yml b/.github/workflows/vllm.yml deleted file mode 100644 index d358349f72c..00000000000 --- a/.github/workflows/vllm.yml +++ /dev/null @@ -1,169 +0,0 @@ -# # Tests layout - -# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance: -# - `tests/trainer` for testing functionality related to `verl/trainer` -# - `tests/models` for testing functionality related to `verl/models` -# - ... - -# There are a few folders with `special_` prefix, created for special purposes: -# - `special_distributed`: unit tests that must run with multiple GPUs -# - `special_e2e`: end-to-end tests with training/generation scripts -# - `special_npu`: tests for NPUs -# - `special_sanity`: a suite of quick sanity tests -# - `special_standalone`: a set of test that are designed to run in dedicated environments - -# Accelerators for tests -# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`. -# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment. - -# # Workflow layout - -# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs: -# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml` -# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml` -# 3. End-to-end tests: `e2e_*.yml` -# 4. Unit tests -# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/**/test_*_on_cpu.py` -# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix. -# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when -# - new workflow yaml is added to `.github/workflows` -# - new tests are added to workflow mentioned in 2. - -name: vllm - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - - v0.* - pull_request: - branches: - - main - - v0.* - paths: - - "**/*.py" - # Other entrypoints - - "!examples/**" - - "!tests/**" - - "!verl/trainer/main_*.py" - - "!verl/trainer/fsdp_sft_trainer.py" - # FSDP - - "!verl/workers/**/*dp_*.py" - # Megatron - - "!verl/workers/**/megatron_*.py" - # SGLang - - "!**/*sglang*" - # Entrypoints - - ".github/workflows/vllm.yml" - - "tests/special_e2e/generation" - - "tests/workers/rollout" - - "verl/trainer/main_generation.py" - - "verl/trainer/config/generation.yaml" - -# Cancel jobs on the same ref if a new one is triggered -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -# Declare permissions just read content. -permissions: - contents: read - -env: - IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:vllm017.dev2" - DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner" - -jobs: - setup: - if: github.repository_owner == 'verl-project' - runs-on: ubuntu-latest - outputs: - runner-label: ${{ steps.create-runner.outputs.runner-label }} - mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }} - steps: - - uses: actions/checkout@v4 - - id: create-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "create" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-image: "${{ env.IMAGE }}" - - vllm: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 35 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install --upgrade "transformers<5.0" - # - name: Download Model to Use - # run: | - # hf download Qwen/Qwen2.5-0.5B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-0.5B-Instruct - # hf download Qwen/Qwen2.5-1.5B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-1.5B-Instruct - # hf download Qwen/Qwen2.5-VL-3B-Instruct --local-dir ${HOME}/models/Qwen/Qwen2.5-VL-3B-Instruct - # hf download OldKingMeister/Qwen2.5-1.5B-Instruct-YaRN --local-dir ${HOME}/models/OldKingMeister/Qwen2.5-1.5B-Instruct-YaRN - # export HF_HUB_OFFLINE=1 - - name: Prepare gsm8k dataset - run: | - ray stop --force - python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k - - name: Test the latest vLLM Rollout async with agent loop - run: | - ROLLOUT_NAME=vllm pytest -svvv tests/experimental/agent_loop - - name: Test vllm server abort functionality - run: | - pytest tests/workers/rollout/rollout_vllm/test_vllm_abort.py -v -s - - vllm_checkpoint_engine: - needs: setup - runs-on: ["${{ needs.setup.outputs.runner-label || 'L20x8' }}"] - timeout-minutes: 35 # Increase this timeout value as needed - env: - HTTP_PROXY: ${{ secrets.PROXY_HTTP }} - HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }} - NO_PROXY: "localhost,127.0.0.1,hf-mirror.com" - HF_ENDPOINT: "https://hf-mirror.com" - HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - name: Install the current repository - run: | - pip3 install pytest-asyncio - pip3 install -r requirements-test.txt - pip3 install --no-deps -e . - pip3 install --upgrade "transformers<5.0" - pip3 install cupy-cuda12x==13.6.0 - - name: Test vLLM ServerAdapter with Checkpoint Engine (NCCL) - run: | - ROLLOUT_NAME=vllm pytest -svvv tests/checkpoint_engine/test_special_server_adapter.py - - name: Test bucketed weight transfer - run: | - pytest -svvv tests/utils/test_bucketed_weight_transfer.py - - cleanup: - runs-on: ubuntu-latest - needs: [setup, vllm, vllm_checkpoint_engine] - if: always() - steps: - - id: destroy-runner - uses: volcengine/vemlp-github-runner@v1 - with: - mode: "destroy" - faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}" - mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" diff --git a/.github/workflows_old/pre-commit.yml b/.github/workflows_old/pre-commit.yml deleted file mode 100644 index 4f6aa4bdf0d..00000000000 --- a/.github/workflows_old/pre-commit.yml +++ /dev/null @@ -1,41 +0,0 @@ -# c.f. https://github.com/pre-commit/action?tab=readme-ov-file#using-this-action -name: pre-commit - -# No need to avoid / cancel lightweight pre-commit jobs -on: - schedule: - - cron: "0 0 * * 0" - pull_request: - push: - branches: - - main - - v0.* - # Allow manual triggering - workflow_dispatch: - -# Declare permissions just read content. -permissions: - contents: read - -jobs: - pre-commit: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.12"] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: ${{ matrix.python-version }} - - name: Install the current repository - run: | - pip install pre-commit hydra-core - pip install --no-deps -e . - - name: Set ruff --output-format=github - run: | - sed -i 's/--output-format=full/--output-format=github/' .pre-commit-config.yaml - git add .pre-commit-config.yaml - # Check "--all-files" by default - - uses: pre-commit/action@v3.0.1 From 6d52ea9aed0052f90c8d643ebf465a0899f43868 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Tue, 31 Mar 2026 20:59:04 +0000 Subject: [PATCH 07/58] integration with AT ARLClient --- verl/trainer/ppo/arctic_rl_client.py | 132 +++++++++++++++++++++++---- 1 file changed, 113 insertions(+), 19 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 8f7e74b21c9..603c4639ce7 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -1,3 +1,5 @@ +import io +import os import torch from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer from deepspeed.utils import OnDevice @@ -10,19 +12,20 @@ from typing import Any from verl.utils.ray_utils import auto_await +USE_ARCTIC_TRAINING_CLIENT = os.environ.get("USE_ARCTIC_TRAINING_CLIENT", "0") == "1" + + def create_arctic_rl_client(): + cls = ArcticRLClientWrapper if USE_ARCTIC_TRAINING_CLIENT else ArcticRLClient4VeRL sched_pg = placement_group([{"GPU": 0, "CPU": 1}]) - arctic_rl_client = ray.remote( + return ray.remote( num_cpus=0, num_gpus=0, scheduling_strategy=PlacementGroupSchedulingStrategy( placement_group=sched_pg, placement_group_capture_child_tasks=True, ), - )(ArcticRLClient4VeRL).remote( - ) - - return arctic_rl_client + )(cls).remote() def create_meta_model(name_or_path: str): model_config = AutoConfig.from_pretrained(name_or_path) @@ -85,42 +88,31 @@ def initialize(self, model_name: str): def generate(self, prompt_ids, sampling_params) -> TokenOutput: prompts = [self.tokenizer.decode(prompt_ids)] - return self.inference_engine.generate( + result = self.inference_engine.generate( prompts=prompts, - sampling_params=sampling_params, ) + return result - def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict): - dss_batch_dict.update(post_process_inputs=post_process_inputs) - + def compute_log_prob(self, dss_batch_dict: dict): # XXX: somehow we need to differentiate which model is this called on ref vs actor - at the moment it's always actor hardcoded entropy, log_probs = self.training_engine.fwd_no_grad(**dss_batch_dict) - # XXX: for some reason no_padding_2_padding expects a 1D tensor - not sure how it'll work for - # bs>1 - # I think it may have to do with tensor.is_nested - different path/logic - # so most likely we need to convert these 2 into TensorDict if entropy is not None: # prior_entropy_shape = entropy.shape entropy = torch.tensor(entropy).squeeze() if log_probs is not None: # prior_log_probs_shape = log_probs.shape log_probs = torch.tensor(log_probs).squeeze() - print(f"arctic_rl_client.compute_log_prob: {entropy.shape=}, {log_probs.shape=}") return entropy, log_probs def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): - dss_batch_dict.update(post_process_inputs=post_process_inputs) - #_ = self.training_engine.forward(**dss_batch_dict, post_process_inputs=post_process_inputs) _ = self.training_engine.forward(**dss_batch_dict) loss, metrics = self.training_engine.backward() self.training_engine.step() - print(f"arctic_rl_client.update_actor: {loss=}") - print(f"arctic_rl_client.update_actor: {metrics=}") return loss.cpu().item(), metrics def destroy(self): @@ -128,3 +120,105 @@ def destroy(self): self.inference_engine.destroy() return + +class ArcticRLClientWrapper: + """Thin wrapper around ArcticTraining's ArcticRLClient that exposes the + same interface as ArcticRLClient4VeRL so it can be used as a drop-in + replacement. + + Set USE_ARCTIC_TRAINING_CLIENT=1 env var to activate. + """ + + def __init__(self): + self._client = None + self.tokenizer = None + + def initialize(self, model_name: str): + from arctic_training.arctic_rl import ArcticRLClient, ArcticRLClientConfig + + config = ArcticRLClientConfig( + host="localhost", + port=7000, + backend="local", + training_gpus=1, + sample_gpus=1, + log_prob_gpus=1, + log_prob_engine="deepspeed", + model_name=model_name, + ds_config={ + "train_micro_batch_size_per_gpu": 1, + "train_batch_size": 1, + "gradient_accumulation_steps": 1, + "sequence_parallel_size": 1, + "zero_optimization": {"stage": 1}, + }, + training_config={ + "optimizer": {"lr": 0.0002, "weight_decay": 0.0, "betas": [0.9, 0.999]}, + "lr_scheduler": {"warmup_ratio": 0.05}, + "training_horizon": 10, + "max_length": 8096, + "model_config": None, + "attn_implementation": "eager", + }, + vllm_config=None, + ) + os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2" + self._client = ArcticRLClient(config) + self.tokenizer = AutoTokenizer.from_pretrained(model_name) + + _default_sampling_params = { + "temperature": 0.0, + "top_p": 1.0, + "top_k": 0, + "max_tokens": 1024, + } + + def generate(self, prompt_ids, sampling_params) -> list: + prompts = [self.tokenizer.decode(prompt_ids)] + if sampling_params is not None and not isinstance(sampling_params, dict): + merged_params = {**self._default_sampling_params, **vars(sampling_params)} + else: + merged_params = {**self._default_sampling_params, **(sampling_params or {})} + return self._client.generate(prompts=prompts, sampling_params=merged_params) + + def compute_log_prob(self, dss_batch_dict: dict): + batch = { + "kwargs": dss_batch_dict, + "context": {"labels": dss_batch_dict["labels"]}, + } + result = self._client.fwd_no_grad(batch, post_processors=["entropy_logprobs"]) + outputs = result.get("model_outputs", result) + + entropy = outputs.get("entropy") + log_probs = outputs.get("log_probs") + + if entropy is not None: + entropy = torch.tensor(entropy).squeeze() + if log_probs is not None: + log_probs = torch.tensor(log_probs).squeeze() + + return entropy, log_probs + + def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): + extra = post_process_inputs.get("extra_inputs", {}) + context = { + "labels": dss_batch_dict["labels"], + "old_logprobs": extra["old_log_probs"], + "advantages": extra["advantages"], + "loss_mask": extra["response_mask"], + } + batch = {"kwargs": dss_batch_dict, "context": context} + + result = self._client.fwd_bwd(batch, loss_fn="grpo") + self._client.step() + + loss = result.get("avg_loss", 0.0) + raw_metrics = result.get("post_process_outputs", {}) + # Caller expects metrics values to be lists (does v[0]) + metrics = {k: v if isinstance(v, list) else [v] for k, v in raw_metrics.items()} + + return loss, metrics + + def destroy(self): + if self._client is not None: + self._client.shutdown() From a3cd7ca383d05712367724bc23e576dc8124d32b Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 1 Apr 2026 19:50:11 +0000 Subject: [PATCH 08/58] align with recent changes from Karthik --- verl/trainer/ppo/arctic_rl_client.py | 37 ++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 603c4639ce7..d7b5e008ee6 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -120,7 +120,8 @@ def destroy(self): self.inference_engine.destroy() return - +# TODO: Once we are happy with this implementation, we can make this the new +# ArcticRLClient4VeRL. class ArcticRLClientWrapper: """Thin wrapper around ArcticTraining's ArcticRLClient that exposes the same interface as ArcticRLClient4VeRL so it can be used as a drop-in @@ -140,6 +141,7 @@ def initialize(self, model_name: str): host="localhost", port=7000, backend="local", + # TODO: Grab GPU counts from VeRL config training_gpus=1, sample_gpus=1, log_prob_gpus=1, @@ -152,6 +154,7 @@ def initialize(self, model_name: str): "sequence_parallel_size": 1, "zero_optimization": {"stage": 1}, }, + # TODO: Grab training config from VeRL config training_config={ "optimizer": {"lr": 0.0002, "weight_decay": 0.0, "betas": [0.9, 0.999]}, "lr_scheduler": {"warmup_ratio": 0.05}, @@ -162,10 +165,17 @@ def initialize(self, model_name: str): }, vllm_config=None, ) - os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2" + + # Feels like a hack, but ArcticRLClient is constructed as a ray remote + # actor with num_gpus=0 - This causes CUDA_VISIBLE_DEVICES to be empty, + # so we need to set it manually. + num_gpus = config.training_gpus + config.sample_gpus + config.log_prob_gpus + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in range(num_gpus)) + self._client = ArcticRLClient(config) self.tokenizer = AutoTokenizer.from_pretrained(model_name) + # TODO: Just for debugging - remove later _default_sampling_params = { "temperature": 0.0, "top_p": 1.0, @@ -174,11 +184,8 @@ def initialize(self, model_name: str): } def generate(self, prompt_ids, sampling_params) -> list: - prompts = [self.tokenizer.decode(prompt_ids)] - if sampling_params is not None and not isinstance(sampling_params, dict): - merged_params = {**self._default_sampling_params, **vars(sampling_params)} - else: - merged_params = {**self._default_sampling_params, **(sampling_params or {})} + prompts = [self.tokenizer.decode(prompt_ids)] # TODO: pass prompt_ids directly + merged_params = {**self._default_sampling_params, **vars(sampling_params)} return self._client.generate(prompts=prompts, sampling_params=merged_params) def compute_log_prob(self, dss_batch_dict: dict): @@ -201,11 +208,21 @@ def compute_log_prob(self, dss_batch_dict: dict): def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): extra = post_process_inputs.get("extra_inputs", {}) + seq_len = dss_batch_dict["input_ids"].shape[-1] + + def _left_pad(t: torch.Tensor) -> torch.Tensor: + """Left-pad a response-only tensor to full sequence length with zeros.""" + pad_len = seq_len - t.shape[-1] + if pad_len <= 0: + return t + pad = torch.zeros(*t.shape[:-1], pad_len, dtype=t.dtype, device=t.device) + return torch.cat([pad, t], dim=-1) + context = { "labels": dss_batch_dict["labels"], - "old_logprobs": extra["old_log_probs"], - "advantages": extra["advantages"], - "loss_mask": extra["response_mask"], + "old_logprobs": _left_pad(extra["old_log_probs"]), + "advantages": _left_pad(extra["advantages"]), + "loss_mask": _left_pad(extra["response_mask"]), } batch = {"kwargs": dss_batch_dict, "context": context} From 276f5bf96b588f4ad375601737023264ef899804 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 1 Apr 2026 19:58:51 +0000 Subject: [PATCH 09/58] add TODO --- verl/trainer/ppo/arctic_rl_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index d7b5e008ee6..89a31cb6e28 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -207,6 +207,7 @@ def compute_log_prob(self, dss_batch_dict: dict): return entropy, log_probs def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): + # TODO: Does this align with the ArcticRLClient4VeRL + dss-platform:verl_integration branch? extra = post_process_inputs.get("extra_inputs", {}) seq_len = dss_batch_dict["input_ids"].shape[-1] From 64cdff5870781852696dc473e797f413c56d0cde Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Thu, 2 Apr 2026 19:09:35 +0000 Subject: [PATCH 10/58] revert changes --- verl/trainer/ppo/arctic_rl_client.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 89a31cb6e28..9e7a25dd3ff 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -88,31 +88,42 @@ def initialize(self, model_name: str): def generate(self, prompt_ids, sampling_params) -> TokenOutput: prompts = [self.tokenizer.decode(prompt_ids)] - result = self.inference_engine.generate( + return self.inference_engine.generate( prompts=prompts, + sampling_params=sampling_params, ) - return result - def compute_log_prob(self, dss_batch_dict: dict): + def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict): + dss_batch_dict.update(post_process_inputs=post_process_inputs) + # XXX: somehow we need to differentiate which model is this called on ref vs actor - at the moment it's always actor hardcoded entropy, log_probs = self.training_engine.fwd_no_grad(**dss_batch_dict) + # XXX: for some reason no_padding_2_padding expects a 1D tensor - not sure how it'll work for + # bs>1 + # I think it may have to do with tensor.is_nested - different path/logic + # so most likely we need to convert these 2 into TensorDict if entropy is not None: # prior_entropy_shape = entropy.shape entropy = torch.tensor(entropy).squeeze() if log_probs is not None: # prior_log_probs_shape = log_probs.shape log_probs = torch.tensor(log_probs).squeeze() + print(f"arctic_rl_client.compute_log_prob: {entropy.shape=}, {log_probs.shape=}") return entropy, log_probs def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): + dss_batch_dict.update(post_process_inputs=post_process_inputs) + #_ = self.training_engine.forward(**dss_batch_dict, post_process_inputs=post_process_inputs) _ = self.training_engine.forward(**dss_batch_dict) loss, metrics = self.training_engine.backward() self.training_engine.step() + print(f"arctic_rl_client.update_actor: {loss=}") + print(f"arctic_rl_client.update_actor: {metrics=}") return loss.cpu().item(), metrics def destroy(self): From 8331cdc4c013cb9bb0a8fdc9b81ddd3ed401c0c0 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Thu, 2 Apr 2026 19:10:24 +0000 Subject: [PATCH 11/58] remove unused imports --- verl/trainer/ppo/arctic_rl_client.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 9e7a25dd3ff..ec7ff7df577 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -1,4 +1,3 @@ -import io import os import torch from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer @@ -8,9 +7,6 @@ from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy from ray.util.placement_group import placement_group from verl.workers.rollout.replica import TokenOutput -from tensordict import TensorDict -from typing import Any -from verl.utils.ray_utils import auto_await USE_ARCTIC_TRAINING_CLIENT = os.environ.get("USE_ARCTIC_TRAINING_CLIENT", "0") == "1" From b1e07094f724480335cb89e06980e64e103101c3 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Fri, 3 Apr 2026 11:51:38 +0000 Subject: [PATCH 12/58] train_mini_natch -> train_global_batch --- verl/trainer/ppo/arctic_rl_client.py | 12 ++++ verl/trainer/ppo/arctic_trainer.py | 102 --------------------------- verl/workers/arctic_workers.py | 85 ++++++++-------------- 3 files changed, 43 insertions(+), 156 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 8f7e74b21c9..7cc55c6e109 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -90,6 +90,18 @@ def generate(self, prompt_ids, sampling_params) -> TokenOutput: sampling_params=sampling_params, ) + # TODO: this should use the reference engine instead of the training engine + def compute_ref_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict): + dss_batch_dict.update(post_process_inputs=post_process_inputs) + entropy, log_probs = self.training_engine.fwd_no_grad(**dss_batch_dict) + if entropy is not None: + entropy = torch.tensor(entropy).squeeze() + if log_probs is not None: + log_probs = torch.tensor(log_probs).squeeze() + print(f"arctic_rl_client.compute_ref_log_prob: {entropy.shape=}, {log_probs.shape=}") + return entropy, log_probs + + def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict): dss_batch_dict.update(post_process_inputs=post_process_inputs) diff --git a/verl/trainer/ppo/arctic_trainer.py b/verl/trainer/ppo/arctic_trainer.py index 2d2a4e95f7b..063497fbfff 100644 --- a/verl/trainer/ppo/arctic_trainer.py +++ b/verl/trainer/ppo/arctic_trainer.py @@ -46,108 +46,6 @@ def __init__( self.wg_kwargs["arctic_rl_client"] = self.rl_client - def init_workers(self): - super().init_workers() - return - # print(f"ArcticPPOTrainer.init_workers: {self.actor_rollout_wg=}") - # print(f"ArcticPPOTrainer.init_workers: {self.ref_policy_wg=}") - # print(f"ArcticPPOTrainer.init_workers: {self.async_rollout_manager=}") - # print(f"ArcticPPOTrainer.init_workers: {self.reward_loop_manager=}") - # print(f"ArcticPPOTrainer.init_workers: {self.checkpoint_manager=}") - - # self.resource_pool_manager.create_resource_pool(use_gpu=self.use_gpu) - - # self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} - - # # create actor and rollout - # actor_role = Role.ActorRolloutRef if Role.ActorRolloutRef in self.role_worker_mapping else Role.ActorRollout - # actor_rollout_resource_pool = self.resource_pool_manager.get_resource_pool(actor_role) - # actor_rollout_cls = RayClassWithInitArgs( - # cls=self.role_worker_mapping[actor_role], - # config=self.config.actor_rollout_ref, - # role=str(actor_role), - # ) - # self.resource_pool_to_cls[actor_rollout_resource_pool][str(actor_role)] = actor_rollout_cls - - # # create reference policy if needed - # # if self.use_reference_policy and Role.RefPolicy in self.role_worker_mapping: - # # resource_pool = self.resource_pool_manager.get_resource_pool(Role.RefPolicy) - # # ref_policy_cls = RayClassWithInitArgs( - # # self.role_worker_mapping[Role.RefPolicy], - # # config=self.config.actor_rollout_ref, - # # role=str(Role.RefPolicy), - # # ) - # # self.resource_pool_to_cls[resource_pool][str(Role.RefPolicy)] = ref_policy_cls - - # # initialize WorkerGroup - # # NOTE: if you want to use a different resource pool for each role, which can support different parallel size, - # # you should not use `create_colocated_worker_cls`. - # # Instead, directly pass different resource pool to different worker groups. - # # See https://github.com/volcengine/verl/blob/master/examples/ray/tutorial.ipynb for more information. - # all_wg = {} - # wg_kwargs = {} # Setting up kwargs for RayWorkerGroup - # if OmegaConf.select(self.config.trainer, "ray_wait_register_center_timeout") is not None: - # wg_kwargs["ray_wait_register_center_timeout"] = self.config.trainer.ray_wait_register_center_timeout - # if OmegaConf.select(self.config.global_profiler, "steps") is not None: - # wg_kwargs["profile_steps"] = OmegaConf.select(self.config.global_profiler, "steps") - # # Only require nsight worker options when tool is nsys - # if OmegaConf.select(self.config.global_profiler, "tool") == "nsys": - # assert ( - # OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") - # is not None - # ), "worker_nsight_options must be set when using nsys with profile_steps" - # wg_kwargs["worker_nsight_options"] = OmegaConf.to_container( - # OmegaConf.select(self.config.global_profiler.global_tool_config.nsys, "worker_nsight_options") - # ) - # wg_kwargs["device_name"] = self.device_name - - # for resource_pool, class_dict in self.resource_pool_to_cls.items(): - # if not class_dict: - # continue - # worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) - # wg_dict = self.ray_worker_group_cls( - # resource_pool=resource_pool, - # ray_cls_with_init=worker_dict_cls, - # use_gpu=self.use_gpu, - # **wg_kwargs, - # ) - # spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) - # all_wg.update(spawn_wg) - - - # self.actor_rollout_wg = all_wg[str(actor_role)] - # self.actor_rollout_wg.init_model() - - # # create reward loop manager - # from verl.experimental.reward_loop import RewardLoopManager - - # # initalize reward loop manager - # # reward model (colocate or standalone): get resource_pool - # # no reward model: resource_pool = None - # resource_pool = self.resource_pool_manager.get_resource_pool(Role.RewardModel) if self.use_rm else None - # self.reward_loop_manager = RewardLoopManager( - # config=self.config, - # rm_resource_pool=resource_pool, - # ) - - # self.async_rollout_mode = True - # from verl.experimental.agent_loop import AgentLoopManager - - # # enable_agent_reward_loop = not self.use_rm or self.config.reward.reward_model.enable_resource_pool - # # reward_loop_worker_handles = self.reward_loop_manager.reward_loop_workers if enable_agent_reward_loop else None - # # self.async_rollout_manager = AgentLoopManager.create( - # # config=self.config, - # # worker_group=self.actor_rollout_wg, - # # rollout_resource_pool=actor_rollout_resource_pool, - # # reward_loop_worker_handles=reward_loop_worker_handles, - # # ) - - # self.ref_policy_wg = self.actor_rollout_wg - # self.checkpoint_manager = self.actor_rollout_wg - # self.async_rollout_manager = self.actor_rollout_wg - - - def destroy(self): # self.actor_rollout_wg.destroy() self.rl_client.destroy.remote() \ No newline at end of file diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 0882870c77e..d715fba701c 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -405,8 +405,8 @@ def _postprocess_output(self, output, *, global_token_num, delta_time, forward_o @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train"), blocking=False) - def train_mini_batch(self, data: TensorDict) -> TensorDict: - """Split a batch into N mini-batches run for multiple epochs + def train_global_batch(self, data: TensorDict) -> TensorDict: + """Train a global batch Args: data: @@ -414,62 +414,39 @@ def train_mini_batch(self, data: TensorDict) -> TensorDict: Returns: """ - batch_size_per_dp = data.shape[0] disable_auto_offload = tu.pop(data, key="disable_auto_offload", default=False) - mini_batch_size = tu.pop(data, key="mini_batch_size", default=None) - num_mini_batch = tu.pop(data, key="num_mini_batch", default=None) - epochs = tu.pop(data, key="epochs", default=1) - seed = tu.pop(data, key="seed", default=42) - dataloader_kwargs = tu.pop(data, key="dataloader_kwargs", default={}) self.engine_config = self.config.engine_config - assert mini_batch_size is not None or num_mini_batch is not None - - mini_batch_size_per_gpu = mini_batch_size - - # make iterator - dataloader = tu.make_iterator( - data, - mini_batch_size=mini_batch_size_per_gpu, - epochs=epochs, - seed=seed, - dataloader_kwargs=dataloader_kwargs, - ) - with ( Timer(name="train_batch", logger=None), ): # update - output_lst = [] - total_num_iterations = data.shape[0] // mini_batch_size_per_gpu * epochs - - for batch_idx, mini_batch_td in enumerate(dataloader): - # add global token num - global_token_num = mini_batch_td["input_ids"].offsets().diff().tolist() # (total_nnz,) - tu.assign_non_tensor( - mini_batch_td, - global_token_num=NonTensorData(global_token_num), - update_lr_scheduler=batch_idx == total_num_iterations - 1, - disable_auto_offload=True, - ) - actor_output = self.train_batch(mini_batch_td) - output_lst.append(actor_output) + global_token_num = data["input_ids"].offsets().diff().tolist() # (total_nnz,) + tu.assign_non_tensor( + data, + global_token_num=NonTensorData(global_token_num), + update_lr_scheduler=True, + disable_auto_offload=disable_auto_offload, + ) + + actor_output = self.train_batch(data) + + output_metrics = tu.get(actor_output, "metrics") - actor_output = [tu.get(output, "metrics") for output in output_lst] metrics = {} - for output in actor_output: - for key, val in output.items(): - print(f"metrics {key=} {val=}") - - # flattn dp and micro batch - if isinstance(val, list): - output[key] = ( - Metric.aggregate_dp(val) - if isinstance(val[0], Metric) - else list(chain.from_iterable(val)) - ) - append_to_dict(metrics, output) + for key, val in output_metrics.items(): + print(f"metrics {key=} {val=}") + + # flattn dp and micro batch + if isinstance(val, list): + output_metrics[key] = ( + Metric.aggregate_dp(val) + if isinstance(val[0], Metric) + else list(chain.from_iterable(val)) + ) + + append_to_dict(metrics, output_metrics) output = tu.get_tensordict(tensor_dict={}, non_tensor_dict={"metrics": metrics}).cpu() @@ -562,7 +539,7 @@ def safe_serialize(obj): policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) post_process_inputs = dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config, extra_inputs=extra_inputs) - print(f"update_actor: {post_process_inputs=}") + # print(f"update_actor: {post_process_inputs=}") # XXX: pass the original batch as post_process_inputs["batch"] - the ppo loss function expects data["prompts"] # it got stripped and is not in dss_batch_dict @@ -793,7 +770,7 @@ def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: self._update_config_params(data) post_process_inputs = prepare_log_prob_extra_inputs(data) - entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) + entropy, log_probs = ray.get(self.arctic_rl_client.compute_ref_log_prob.remote(dss_batch_dict, post_process_inputs)) batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) model_output = batch_output.pop("model_output", {}) @@ -859,7 +836,7 @@ def compute_log_prob(self, data: TensorDict) -> TensorDict: @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) @DistProfiler.annotate(color="red", role="actor_update") def update_actor(self, data: TensorDict) -> TensorDict: - output = self.actor.train_mini_batch(data=data) + output = self.actor.train_global_batch(data=data) return output.cpu() if output is not None else None @@ -891,16 +868,16 @@ async def update_weights(self, global_steps: int = None): # TODO: Rollout API Begin @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="rollout")) async def generate_sequences(self, batch: DataProto): - print(f"{batch.non_tensor_batch=}") + # print(f"{batch.non_tensor_batch=}") raw_prompts = list(batch.non_tensor_batch["raw_prompt"]) - print(f"{raw_prompts=}") + # print(f"{raw_prompts=}") prompts = self.tokenizer.apply_chat_template( raw_prompts, add_generation_prompt=True, tokenize=False, ) # import pdb; pdb.set_trace() - print(f"prompts: {prompts}") + # print(f"prompts: {prompts}") metrics = {} gen_batch_output = self.arctic_inference_engine.generate(prompts=prompts) From e01c9dad1d992bff6b902c06b19cf23469b7a8d2 Mon Sep 17 00:00:00 2001 From: Tunji Ruwase Date: Fri, 3 Apr 2026 16:04:59 -0400 Subject: [PATCH 13/58] Merge ZoRRO work (#13) --- examples/arctic_rl/run_arctic_gsm8k_grpo.sh | 14 ++- verl/trainer/ppo/arctic_rl_client.py | 23 ++++- verl/trainer/ppo/arctic_trainer.py | 2 +- verl/workers/arctic_workers.py | 92 ++++++++++++-------- verl/workers/utils/padding.py | 94 ++++++++++++++++++++- 5 files changed, 181 insertions(+), 44 deletions(-) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh index 44cc760c708..81868386d91 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh @@ -34,13 +34,24 @@ USE_ARCTIC_RL=True experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_at${USE_ARCTIC_RL}" +gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) +if [[ $gpu_name == *"H200"* ]]; then + echo "Running on Hopper" + flash_attention_v=flash_attention_3 +elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then + echo "Running on Blackwell" + flash_attention_v=flash_attention_2 +else + echo "Running on unknown: $gpu_name; don't know which FA version to use" +fi + python3 -m verl.trainer.main_ppo \ algorithm.adv_estimator=grpo \ data.train_files=/code/shared/gsm8k/train.parquet \ data.val_files=/code/shared/gsm8k/test.parquet \ data.train_batch_size=${BSZ} \ data.max_prompt_length=64 \ - data.max_response_length=512 \ + data.max_response_length=96 \ data.filter_overlong_prompts=True \ data.truncation='error' \ data.shuffle=False \ @@ -56,6 +67,7 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.kl_loss_type=low_var_kl \ actor_rollout_ref.actor.entropy_coeff=0 \ actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ actor_rollout_ref.actor.strategy=${STRATEGY} \ actor_rollout_ref.actor.fsdp_config.param_offload=False \ actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 7cc55c6e109..50d96576520 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -10,7 +10,7 @@ from typing import Any from verl.utils.ray_utils import auto_await -def create_arctic_rl_client(): +def create_arctic_rl_client(config): sched_pg = placement_group([{"GPU": 0, "CPU": 1}]) arctic_rl_client = ray.remote( num_cpus=0, @@ -19,7 +19,7 @@ def create_arctic_rl_client(): placement_group=sched_pg, placement_group_capture_child_tasks=True, ), - )(ArcticRLClient4VeRL).remote( + )(ArcticRLClient4VeRL).remote(config ) return arctic_rl_client @@ -31,11 +31,18 @@ def create_meta_model(name_or_path: str): return meta_model class ArcticRLClient4VeRL: - def __init__(self): + def __init__(self, config): + """ + config: verl's full config + """ + self.config = config + #print(f"ArcticRLClient4VeRL {config=}") + self.arctic_inference_client = DSSInferenceClient(dss_server_url="http://localhost:7000") self.arctic_training_client = DSSTrainingClient(dss_server_url="http://localhost:7000") self.arctic_log_prob_client = DSSLogProbClient(dss_server_url="http://localhost:7000") + def initialize(self, model_name: str): vllm_config = { "temperature": 0.0, @@ -63,6 +70,14 @@ def initialize(self, model_name: str): "stage": 1, }, } + + # currently verl wants '+' before the setting, i.e. +actor_rollout_ref.model.override_config.attn_implementation=flash_attention_3 + attn_implementation = self.config.actor_rollout_ref.model.override_config.get('attn_implementation', 'eager') + if attn_implementation == "eager": + raise ValueError("set actor_rollout_ref.model.override_config.attn_implementation to some variant of flash attention") + + #attn_implementation="flash_attention_3" + training_config = { "optimizer": { "lr": 0.0002, @@ -73,7 +88,7 @@ def initialize(self, model_name: str): "training_horizon": 10, "max_length": 8096, "model_config": None, - "attn_implementation": "eager", + "attn_implementation": attn_implementation, } self.training_engine = self.arctic_training_client.initialize( diff --git a/verl/trainer/ppo/arctic_trainer.py b/verl/trainer/ppo/arctic_trainer.py index 063497fbfff..f4e0cc0d187 100644 --- a/verl/trainer/ppo/arctic_trainer.py +++ b/verl/trainer/ppo/arctic_trainer.py @@ -41,7 +41,7 @@ def __init__( device_name=device_name) self.use_gpu = False - self.rl_client = create_arctic_rl_client() + self.rl_client = create_arctic_rl_client(config=config) self.rl_client.initialize.remote(model_name="Qwen/Qwen3-0.6B") self.wg_kwargs["arctic_rl_client"] = self.rl_client diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index d715fba701c..b099b68ea35 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -181,7 +181,7 @@ def prepare_extra_inputs(data: TensorDict) -> dict: prompts=data["prompts"], responses=data["responses"], attention_mask=data["attention_mask"], - max_response_len=data["max_response_len"], + max_token_len_per_gpu=data["max_token_len_per_gpu"], global_batch_size=data["global_batch_size"], response_mask=data["response_mask"], old_log_probs=data["old_log_probs"], @@ -260,7 +260,7 @@ class TrainingWorker(Worker, DistProfilerExtension): and do not provide exact APIs as Tinker does. But this can be added in the future. """ - def __init__(self, config: TrainingWorkerConfig, actor_config: ActorConfig, arctic_rl_client): + def __init__(self, config: TrainingWorkerConfig, actor_config: ActorConfig, arctic_rl_client, tokenizer): Worker.__init__(self) from verl.workers.engine import BaseEngine, EngineRegistry @@ -271,6 +271,7 @@ def __init__(self, config: TrainingWorkerConfig, actor_config: ActorConfig, arct self.actor_config = actor_config self.arctic_rl_client = arctic_rl_client + self.tokenizer = tokenizer self.model_config = self.config.model_config self.engine_config = self.config.engine_config @@ -278,6 +279,8 @@ def __init__(self, config: TrainingWorkerConfig, actor_config: ActorConfig, arct self.checkpoint_config = self.config.checkpoint_config self.device_name = get_device_name() + print(f"{self.engine_config=}") + if self.engine_config is None: assert self.optimizer_config is None if self.config.auto_select_engine_optim_fn is None: @@ -487,19 +490,57 @@ def train_batch(self, data: TensorDict) -> TensorDict: # batch = chunk_tensordict(data, 1) print(f"update_actor data: {data}") + # XXX: fix me + padding_token = 100 + input_ids = data['input_ids'] + position_ids = data['position_ids'] + #input_ids = input_ids.unbind() + + from verl.workers.utils.padding import no_padding_2_padding_prompt_response + # XXX: move to init + if self.tokenizer.pad_token_id is None: + self.tokenizer.pad_token_id = self.tokenizer.eos_token_id + + input_ids, max_prompt_len, max_response_len = no_padding_2_padding_prompt_response(tensor=input_ids, data=data, pad_token_id=self.tokenizer.pad_token_id) + # XXX: 0 pad on pos ids is odd, check the original - perhaps need to re-build pos ids? + position_ids, _, _= no_padding_2_padding_prompt_response(tensor=position_ids, data=data, pad_token_id=0) + print(f"{input_ids.shape=}") + print(f"{input_ids=}") + + #input_ids = torch.nested.to_padded_tensor(input_ids, padding=4.2) + #position_ids = torch.nested.to_padded_tensor(position_ids, padding=4.2) + + print(f"{data['attention_mask'].shape=}") + print(f"{data['attention_mask']=}") + print(f"{input_ids.shape=}") + print(f"{input_ids=}") + print(f"{position_ids.shape=}") + print(f"{position_ids=}") # XXX: fixme # batch = batch[0] - dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + #dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + # print(f"{dss_batch_dict=}") + #print(f"{output_args=}") + #import pdb; pdb.set_trace() + + dss_batch_dict = dict( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=data['attention_mask'], + labels=input_ids, + ) print(f"{dss_batch_dict=}") - print(f"{output_args=}") - # import pdb; pdb.set_trace() + + + rollout_n = self.actor_config.rollout_n + max_token_len_per_gpu = self.actor_config.ppo_max_token_len_per_gpu # we need to serialize the config object to dict # dataclasses.asdict only returns keys that are defined at init (vars will do more) - but perhaps we want `asdict`? actor_config_as_dict = vars(self.actor_config) print(f"update_actor: {self.actor_config=}") - print(f"update_actor: {actor_config_as_dict}") + print(f"update_actor: {actor_config_as_dict=}") import json def safe_serialize(obj): return json.loads(json.dumps(obj, default=lambda o: None)) @@ -507,35 +548,10 @@ def safe_serialize(obj): actor_config_as_dict = safe_serialize(actor_config_as_dict) - # pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - # batch_size = data["input_ids"].shape[0] - # seq_len_effective = data["input_ids"].offsets().diff() - # max_seq_len = max(seq_len_effective) - # ready_input_ids = torch.nested.to_padded_tensor( - # data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) - # ) - # ready_position_ids = torch.nested.to_padded_tensor( - # data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) - # ) - # extra_inputs = dict( - # prompts=data["prompts"], - # responses=data["responses"], - # attention_mask=data["attention_mask"], - # max_response_len=data["max_response_len"], - # global_batch_size=data["global_batch_size"], - # response_mask=data["response_mask"], - # old_log_probs=data["old_log_probs"], - # advantages=data["advantages"], - # ref_log_prob=data["ref_log_prob"], - # rollout_is_weights=data.get("rollout_is_weights", None), - # ready_input_ids=ready_input_ids, - # ready_position_ids=ready_position_ids, - # ready_labels=ready_input_ids, - # # =batch[""], - # ) - # extra_inputs["batch_num_tokens"] = data["loss_mask"].sum() - extra_inputs = prepare_extra_inputs(data) + extra_inputs["rollout_n"] = rollout_n + extra_inputs["max_prompt_len"] = max_prompt_len + extra_inputs["max_response_len"] = max_response_len policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) post_process_inputs = dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config, extra_inputs=extra_inputs) @@ -691,7 +707,9 @@ def __init__(self, config: DictConfig, role: str, **kwargs): assert self.config.rollout.log_prob_micro_batch_size_per_gpu is not None assert self.config.actor.ppo_micro_batch_size_per_gpu is not None - self.actor = TrainingWorker(config=actor_training_config, actor_config=actor_config, arctic_rl_client=self.arctic_rl_client) + # XXX: fix me - duplicated in _init_engines and model hardcoded + self.tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") + self.actor = TrainingWorker(config=actor_training_config, actor_config=actor_config, arctic_rl_client=self.arctic_rl_client, tokenizer=self.tokenizer ) self.actor.reset() self.loss_fn = partial(ppo_loss, config=actor_config) @@ -758,7 +776,7 @@ def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: # import pdb; pdb.set_trace() # from verl.utils.tensordict_utils import chunk_tensordict # batch = chunk_tensordict(data, 1) - # print(f"compute_ref_log_prob data: {data}") + print(f"compute_ref_log_prob data: {data}") dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) # print(f"{dss_batch_dict=}") # import pdb; pdb.set_trace() @@ -805,7 +823,7 @@ def compute_log_prob(self, data: TensorDict) -> TensorDict: # import pdb; pdb.set_trace() # from verl.utils.tensordict_utils import chunk_tensordict # batch = chunk_tensordict(data, 1) - # print(f"compute_log_prob data: {data}") + print(f"compute_log_prob data: {data}") dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) # print(f"{dss_batch_dict=}") # import pdb; pdb.set_trace() diff --git a/verl/workers/utils/padding.py b/verl/workers/utils/padding.py index 16242e7731f..69ed8de90f5 100644 --- a/verl/workers/utils/padding.py +++ b/verl/workers/utils/padding.py @@ -94,13 +94,19 @@ def no_padding_2_padding(tensor: torch.Tensor, data: TensorDict) -> torch.Tensor Returns: tensor: sliced response tensor of shape [bsz, max_response_len] """ + # print(f"{tensor.is_nested=}") values = tensor.values() if tensor.is_nested else tensor prompt_ids = data["prompts"] response_ids = data["responses"] attention_mask = data["attention_mask"] + # print(f"{prompt_ids.shape=}") + # print(f"{response_ids.shape=}") + # print(f"{attention_mask.shape=}") + # print(f"{attention_mask=}") max_response_len = tu.get_non_tensor_data(data=data, key="max_response_len", default=-1) + # print(f"{prompt_ids.is_nested=}") if prompt_ids.is_nested: prompt_lens = prompt_ids.offsets().diff() response_lens = response_ids.offsets().diff() @@ -114,7 +120,15 @@ def no_padding_2_padding(tensor: torch.Tensor, data: TensorDict) -> torch.Tensor sequence_lens = prompt_lens + response_lens sequence_offsets = sequence_lens.cumsum(dim=0) - assert sequence_offsets[-1].item() == values.shape[0] + print(f"{data=}") + print(f"{prompt_lens=}") + print(f"{response_lens=}") + print(f"{response_lens=}") + print(f"{max_response_len=}") + print(f"{sequence_offsets=}") + print(f"{values=}") + print(f"{values.shape=}") + assert sequence_offsets[-1].item() == values.shape[0], f"{sequence_offsets[-1].item()} != {values.shape[0]}" response_list = [] for resp_len, seq_offset in zip(response_lens, sequence_offsets, strict=True): @@ -124,3 +138,81 @@ def no_padding_2_padding(tensor: torch.Tensor, data: TensorDict) -> torch.Tensor output = torch.stack(response_list, dim=0) return output + + + +def no_padding_2_padding_prompt_response(tensor: torch.Tensor, data: TensorDict, pad_token_id) -> torch.Tensor: + """Convert jagged tensor into a left padded prompt and right padded prompt of [bsz, max_response_len], which looks like + tensor([ + [pad...prompt | response...pad], + [pad...prompt | response...pad], + [pad...prompt | response...pad] + ]) + + Args: + tensor: a nested tensor or a 1D tensor in shape (total_nnz,), + total_nnz is the total number of tokens across all sequences in the batch + data: TensorDict with "prompts", "responses", "attention_mask" + pad_token_id: token to pad with + + Returns: + tensor: sliced prompt+response tensor of shape [bsz, max_response_len] w/ left and right padding + + """ + # print(f"{tensor.is_nested=}") + values = tensor.values() if tensor.is_nested else tensor + prompt_ids = data["prompts"] + response_ids = data["responses"] + attention_mask = data["attention_mask"] + # print(f"{prompt_ids.shape=}") + # print(f"{response_ids.shape=}") + # print(f"{attention_mask.shape=}") + # print(f"{attention_mask=}") + + max_prompt_len = tu.get_non_tensor_data(data=data, key="max_prompt_len", default=-1) + max_response_len = tu.get_non_tensor_data(data=data, key="max_response_len", default=-1) + print(f"data {max_prompt_len=}") + print(f"data {max_response_len=}") + + # print(f"{prompt_ids.is_nested=}") + if prompt_ids.is_nested: + prompt_lens = prompt_ids.offsets().diff() + response_lens = response_ids.offsets().diff() + if max_prompt_len < 0: + max_prompt_len = prompt_lens.max().item() + if max_response_len < 0: + max_response_len = response_lens.max().item() + else: + assert not attention_mask.is_nested + prompt_lens = attention_mask[:, : prompt_ids.shape[1]].sum(dim=1) + response_lens = attention_mask[:, prompt_ids.shape[1] :].sum(dim=1) + max_prompt_len = prompt_ids.shape[1] + max_response_len = response_ids.shape[1] + + sequence_lens = prompt_lens + response_lens + sequence_offsets = sequence_lens.cumsum(dim=0) + print(f"{data=}") + print(f"{prompt_lens=}") + print(f"{response_lens=}") + print(f"{max_prompt_len=}") + print(f"{max_response_len=}") + print(f"{sequence_offsets=}") + print(f"{values=}") + print(f"{values.shape=}") + assert sequence_offsets[-1].item() == values.shape[0], f"{sequence_offsets[-1].item()} != {values.shape[0]}" + + input_ids_list = [] + for prompt_len, resp_len, seq_offset in zip(prompt_lens, response_lens, sequence_offsets, strict=True): + prompt_pad_size = max_prompt_len - prompt_len + response_pad_size = max_response_len - resp_len + prompt = values[seq_offset - prompt_len - resp_len: seq_offset - resp_len] + response = values[seq_offset - resp_len: seq_offset] + prompt_padded_left = F.pad(prompt, (prompt_pad_size, 0), value=pad_token_id) + response_padded_right = F.pad(response, (0, response_pad_size), value=pad_token_id) + input_ids_list.append(torch.cat((prompt_padded_left, response_padded_right))) + + output = torch.stack(input_ids_list, dim=0) + #print(f"{output=}") + return output, max_prompt_len, max_response_len + + From 6b1889eb7340f69b503f90108207c462527bdf24 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Fri, 3 Apr 2026 20:35:01 +0000 Subject: [PATCH 14/58] Debugging multi-gpu --- examples/arctic_rl/run_arctic_gsm8k_grpo.sh | 2 +- examples/arctic_rl/run_gsm8k_grpo.sh | 2 +- verl/workers/arctic_workers.py | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh index 44cc760c708..5d77b10dd95 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh @@ -13,7 +13,7 @@ BSZ=2 MBS=2 UBS=2 ROLL_N=2 -MAX_STEPS=1 +MAX_STEPS=4 # LR=0 LR=1e-6 LOGGER=console diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh index f65b0f55b24..1317d8091d9 100755 --- a/examples/arctic_rl/run_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_gsm8k_grpo.sh @@ -10,7 +10,7 @@ BSZ=2 MBS=2 UBS=2 ROLL_N=2 -MAX_STEPS=1 +MAX_STEPS=4 # LR=0 LR=1e-6 LOGGER=console diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index d715fba701c..d6003589846 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -218,6 +218,8 @@ def prepare_log_prob_extra_inputs(data: TensorDict) -> dict: cu_seqlens=data["input_ids"].offsets() ) + print(f"prepare_log_prob_extra_inputs: {data['input_ids'].shape=} {ready_input_ids.shape=} {data['position_ids'].shape=} {ready_position_ids.shape=}") + return extra_inputs @@ -436,7 +438,7 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: metrics = {} for key, val in output_metrics.items(): - print(f"metrics {key=} {val=}") + # print(f"metrics {key=} {val=}") # flattn dp and micro batch if isinstance(val, list): @@ -556,8 +558,8 @@ def safe_serialize(obj): loss, metrics = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) # output = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) - print(f"update_actor: {loss=}") - print(f"update_actor: {metrics=}") + # print(f"update_actor: {loss=}") + # print(f"update_actor: {metrics=}") from verl.utils.metric import AggregationType, Metric @@ -577,7 +579,7 @@ def safe_serialize(obj): # 'grad_norm': None, # } - print(f"{data=}") + # print(f"{data=}") print(f"{data["input_ids"].shape=}") model_output = { # XXX: fix me - made a copy of existing same shape tensor for now From 2a15fdcdb34d0f9f8effab57b4b9c816628d7a88 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 3 Apr 2026 20:35:57 +0000 Subject: [PATCH 15/58] missing Signed-off-by: Stas Bekman --- examples/arctic_rl/debug_at_gsm8k_grpo.sh | 97 +++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100755 examples/arctic_rl/debug_at_gsm8k_grpo.sh diff --git a/examples/arctic_rl/debug_at_gsm8k_grpo.sh b/examples/arctic_rl/debug_at_gsm8k_grpo.sh new file mode 100755 index 00000000000..e1c2fbf581f --- /dev/null +++ b/examples/arctic_rl/debug_at_gsm8k_grpo.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +# we want to make sure this runs on non-gpu client +export CUDA_VISIBLE_DEVICES= + +# BSZ=1024 +BSZ=1 +MBS=1 +UBS=1 +ROLL_N=4 +MAX_STEPS=4 +# LR=0 +LR=1e-6 +LOGGER=console +# LOGGER="['console','wandb']" +USE_KL_LOSS=True +# USE_KL_LOSS=False +REMOVE_PADDING=True +# REMOVE_PADDING=False +MODEL="Qwen/Qwen3-0.6B" +# STRATEGY="fsdp" +STRATEGY="fsdp2" +PYTHONUNBUFFERED=1 +HYDRA_FULL_ERROR=1 +USE_LEGACY_WORKER_IMPL=disable +USE_ARCTIC_RL=True +NGPU_PER_NODE=1 +ROLLOUT_NAME=arctic # entry point into ArcticRL +experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_at${USE_ARCTIC_RL}" + +gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) +if [[ $gpu_name == *"H200"* ]]; then + echo "Running on Hopper" + flash_attention_v=flash_attention_3 +elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then + echo "Running on Blackwell" + flash_attention_v=flash_attention_2 +else + echo "Running on unknown: $gpu_name; don't know which FA version to use" +fi + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=/code/shared/gsm8k/train.parquet \ + data.val_files=/code/shared/gsm8k/test.parquet \ + data.train_batch_size=${BSZ} \ + data.max_prompt_length=64 \ + data.max_response_length=96 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + reward.num_workers=1 \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.model.path=${MODEL} \ + actor_rollout_ref.actor.optim.lr=${LR} \ + actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ + actor_rollout_ref.actor.strategy=${STRATEGY} \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.n=${ROLL_N} \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.ref.fsdp_config.param_offload=False \ + actor_rollout_ref.ref.strategy=${STRATEGY} \ + algorithm.use_kl_in_reward=False \ + trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ + trainer.use_arctic_rl=${USE_ARCTIC_RL} \ + trainer.critic_warmup=0 \ + trainer.logger=${LOGGER} \ + trainer.experiment_name=${experiment_name} \ + trainer.project_name='verl_arctic_grpo_gsm8k' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_training_steps=${MAX_STEPS} \ + trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log + + # trainer.total_training_steps=${MAX_STEPS} \ From 928b66b134224320f0204e8f7ba4b3f2ad9783ee Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Fri, 3 Apr 2026 22:40:24 +0000 Subject: [PATCH 16/58] update to work with recent arl changes --- verl/trainer/ppo/arctic_rl_client.py | 36 ++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 111493f06bd..b31370e7466 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -11,7 +11,7 @@ USE_ARCTIC_TRAINING_CLIENT = os.environ.get("USE_ARCTIC_TRAINING_CLIENT", "0") == "1" -def create_arctic_rl_client(): +def create_arctic_rl_client(config): cls = ArcticRLClientWrapper if USE_ARCTIC_TRAINING_CLIENT else ArcticRLClient4VeRL sched_pg = placement_group([{"GPU": 0, "CPU": 1}]) return ray.remote( @@ -219,10 +219,13 @@ def initialize(self, model_name: str): def generate(self, prompt_ids, sampling_params) -> list: prompts = [self.tokenizer.decode(prompt_ids)] # TODO: pass prompt_ids directly - merged_params = {**self._default_sampling_params, **vars(sampling_params)} + merged_params = {**self._default_sampling_params, **sampling_params} return self._client.generate(prompts=prompts, sampling_params=merged_params) - def compute_log_prob(self, dss_batch_dict: dict): + def compute_ref_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict = None): + return self.compute_log_prob(dss_batch_dict, post_process_inputs) + + def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict = None): batch = { "kwargs": dss_batch_dict, "context": {"labels": dss_batch_dict["labels"]}, @@ -234,9 +237,32 @@ def compute_log_prob(self, dss_batch_dict: dict): log_probs = outputs.get("log_probs") if entropy is not None: - entropy = torch.tensor(entropy).squeeze() + entropy = torch.tensor(entropy) if log_probs is not None: - log_probs = torch.tensor(log_probs).squeeze() + log_probs = torch.tensor(log_probs) + + # TODO: AI fix to resolve problems after merge into arl branch. Need to verify correctness of this. + # The model returns packed (1, total_nnz) tensors, but downstream + # rm_padding expects padded (bsz, max_seq_len). Unpack and re-pad. + if post_process_inputs is not None: + cu_seqlens = post_process_inputs["cu_seqlens"] + seq_lengths = cu_seqlens.diff() + bsz = seq_lengths.shape[0] + max_seq_len = int(seq_lengths.max()) + + def _packed_to_padded(t): + flat = t.reshape(-1) + padded = torch.zeros(bsz, max_seq_len, dtype=flat.dtype, device=flat.device) + for i in range(bsz): + start = int(cu_seqlens[i]) + length = int(seq_lengths[i]) + padded[i, :length] = flat[start:start + length] + return padded + + if entropy is not None: + entropy = _packed_to_padded(entropy) + if log_probs is not None: + log_probs = _packed_to_padded(log_probs) return entropy, log_probs From f9ba22eb1a7dc9e09208daa57926226067edb8d5 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Mon, 6 Apr 2026 18:48:00 +0000 Subject: [PATCH 17/58] Multi-GPU --- verl/trainer/ppo/arctic_rl_client.py | 2 - verl/workers/arctic_workers.py | 117 ++++++++++----------------- 2 files changed, 43 insertions(+), 76 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 50d96576520..491b2606abb 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -128,10 +128,8 @@ def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict): # I think it may have to do with tensor.is_nested - different path/logic # so most likely we need to convert these 2 into TensorDict if entropy is not None: - # prior_entropy_shape = entropy.shape entropy = torch.tensor(entropy).squeeze() if log_probs is not None: - # prior_log_probs_shape = log_probs.shape log_probs = torch.tensor(log_probs).squeeze() print(f"arctic_rl_client.compute_log_prob: {entropy.shape=}, {log_probs.shape=}") return entropy, log_probs diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index d52115b5172..971e2e7ddcc 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -35,6 +35,7 @@ from omegaconf import DictConfig, open_dict from tensordict import NonTensorData, TensorDict from torch.distributed.device_mesh import init_device_mesh +import torch.nn.functional as F try: from verl.workers.engine.mindspeed.transformer_impl import repatch @@ -165,17 +166,10 @@ def prepare_model_inputs_remove_padding(micro_batch: TensorDict): -def prepare_extra_inputs(data: TensorDict) -> dict: +def prepare_extra_inputs(data: TensorDict, max_prompt_len: int) -> dict: pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - batch_size = data["input_ids"].shape[0] - seq_len_effective = data["input_ids"].offsets().diff() - max_seq_len = max(seq_len_effective) - ready_input_ids = torch.nested.to_padded_tensor( - data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) - ) - ready_position_ids = torch.nested.to_padded_tensor( - data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) - ) + tensors_to_pad = ["advantages", "ref_log_prob", "old_log_probs", "response_mask"] + padded_tensors = {k: F.pad(data[k], pad=(max_prompt_len, 0), value=pad_token_id) for k in tensors_to_pad} extra_inputs = dict( prompts=data["prompts"], @@ -183,47 +177,18 @@ def prepare_extra_inputs(data: TensorDict) -> dict: attention_mask=data["attention_mask"], max_token_len_per_gpu=data["max_token_len_per_gpu"], global_batch_size=data["global_batch_size"], - response_mask=data["response_mask"], - old_log_probs=data["old_log_probs"], - advantages=data["advantages"], - ref_log_prob=data["ref_log_prob"], + response_mask=padded_tensors["response_mask"], + old_log_probs=padded_tensors["old_log_probs"], + advantages=padded_tensors["advantages"], + ref_log_prob=padded_tensors["ref_log_prob"], rollout_is_weights=data.get("rollout_is_weights", None), batch_num_tokens=data["loss_mask"].sum(), - ready_input_ids=ready_input_ids, - ready_position_ids=ready_position_ids, - ready_labels=ready_input_ids, - cu_seqlens=data["input_ids"].offsets(), ) - return extra_inputs -def prepare_log_prob_extra_inputs(data: TensorDict) -> dict: - pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - batch_size = data["input_ids"].shape[0] - seq_len_effective = data["input_ids"].offsets().diff() - max_seq_len = max(seq_len_effective) - - ready_input_ids = torch.nested.to_padded_tensor( - data["input_ids"], padding=pad_token_id, output_size=(batch_size, max_seq_len) - ) - ready_position_ids = torch.nested.to_padded_tensor( - data["position_ids"], padding=0, output_size=(batch_size, max_seq_len) - ) - - extra_inputs = dict( - ready_input_ids=ready_input_ids, - ready_position_ids=ready_position_ids, - ready_labels=ready_input_ids, - cu_seqlens=data["input_ids"].offsets() - ) - - print(f"prepare_log_prob_extra_inputs: {data['input_ids'].shape=} {ready_input_ids.shape=} {data['position_ids'].shape=} {ready_position_ids.shape=}") - return extra_inputs - - -def rm_padding(data: TensorDict, tensor: Tensor) -> Tensor: +def make_njt(data: TensorDict, tensor: Tensor) -> Tensor: cu_seqlens = data["input_ids"].offsets() seq_lengths = cu_seqlens.diff() # (bsz,) starts = torch.zeros_like(seq_lengths, dtype=torch.int64) # (bsz,) @@ -233,8 +198,8 @@ def rm_padding(data: TensorDict, tensor: Tensor) -> Tensor: return tensor def postprocess_log_prob_output(data: TensorDict, entropy: Tensor, log_probs: Tensor) -> TensorDict: - x_entropy = rm_padding(data, entropy) - x_log_probs = rm_padding(data, log_probs) + x_entropy = make_njt(data, entropy) + x_log_probs = make_njt(data, log_probs) print(f"postprocess_log_prob_output: {x_entropy.shape=} {x_log_probs.shape=} {entropy.shape=} {log_probs.shape=}") @@ -255,6 +220,29 @@ def postprocess_log_prob_output(data: TensorDict, entropy: Tensor, log_probs: Te return postprocess_batch_func(output_lst=output_lst, indices=None, data=data) +def prepare_padded_dss_batch_dict(data: TensorDict, pad_token_id) -> dict: + input_ids = data['input_ids'] + position_ids = data['position_ids'] + + from verl.workers.utils.padding import no_padding_2_padding_prompt_response + orig_iput_ids_shape = input_ids.shape + orig_position_ids_shape = position_ids.shape + input_ids, _, _ = no_padding_2_padding_prompt_response(tensor=input_ids, data=data, pad_token_id=pad_token_id) + # XXX: 0 pad on pos ids is odd, check the original - perhaps need to re-build pos ids? + position_ids, _, _= no_padding_2_padding_prompt_response(tensor=position_ids, data=data, pad_token_id=0) + attention_mask = data['attention_mask'] + + print(f"{input_ids.shape=} {position_ids.shape=} {attention_mask.shape=} {orig_iput_ids_shape=} {orig_position_ids_shape=}") + + dss_batch_dict = dict( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + labels=input_ids, + ) + + return dss_batch_dict + class TrainingWorker(Worker, DistProfilerExtension): """ TrainingWorker provides a Tinker-like API (https://thinkingmachines.ai/tinker/) as a RayWorkerGroup @@ -549,8 +537,7 @@ def safe_serialize(obj): #actor_config_as_dict = safe_serialize(self.actor_config) actor_config_as_dict = safe_serialize(actor_config_as_dict) - - extra_inputs = prepare_extra_inputs(data) + extra_inputs = prepare_extra_inputs(data, max_prompt_len) extra_inputs["rollout_n"] = rollout_n extra_inputs["max_prompt_len"] = max_prompt_len extra_inputs["max_response_len"] = max_response_len @@ -779,7 +766,9 @@ def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: # from verl.utils.tensordict_utils import chunk_tensordict # batch = chunk_tensordict(data, 1) print(f"compute_ref_log_prob data: {data}") - dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) + dss_batch_dict = prepare_padded_dss_batch_dict(data, pad_token_id) # print(f"{dss_batch_dict=}") # import pdb; pdb.set_trace() # self.dss_training_engine.forward(**dss_batch_dict) @@ -789,7 +778,7 @@ def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: # log_prob = self._loaded_dump_data["full_log_prob"] self._update_config_params(data) - post_process_inputs = prepare_log_prob_extra_inputs(data) + post_process_inputs = dict() entropy, log_probs = ray.get(self.arctic_rl_client.compute_ref_log_prob.remote(dss_batch_dict, post_process_inputs)) batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) @@ -800,24 +789,9 @@ def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: } final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) - - # metrics = { - # "mfu": 0.0, - # "loss": 1.0, - # "batch_size": 1, - # } - - # model_output = { - # "log_probs": log_probs, - # } - - # final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) - return final_output - - # TODO: Actor API Begin @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") @@ -826,17 +800,12 @@ def compute_log_prob(self, data: TensorDict) -> TensorDict: # from verl.utils.tensordict_utils import chunk_tensordict # batch = chunk_tensordict(data, 1) print(f"compute_log_prob data: {data}") - dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) - # print(f"{dss_batch_dict=}") - # import pdb; pdb.set_trace() - # self.dss_training_engine.forward(**dss_batch_dict) - # loss = self.dss_training_engine.backward() - # print(f"loss: {loss}") - # import pdb; pdb.set_trace() - # log_prob = self._loaded_dump_data["full_log_prob"] + # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) + dss_batch_dict = prepare_padded_dss_batch_dict(data, pad_token_id) self._update_config_params(data) - post_process_inputs = prepare_log_prob_extra_inputs(data) + post_process_inputs = dict() # print(f"compute_log_prob: {post_process_inputs=}") entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) From febd2854aefdd01e8286fc403aee946f8b8f0489 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 6 Apr 2026 23:22:11 +0000 Subject: [PATCH 18/58] merge Signed-off-by: Stas Bekman --- verl/workers/arctic_workers.py | 35 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 971e2e7ddcc..8ad5a9b583b 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -166,11 +166,13 @@ def prepare_model_inputs_remove_padding(micro_batch: TensorDict): -def prepare_extra_inputs(data: TensorDict, max_prompt_len: int) -> dict: +def prepare_extra_inputs(data: TensorDict, max_prompt_len: int, pad_to_prompt_len=True) -> dict: pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) tensors_to_pad = ["advantages", "ref_log_prob", "old_log_probs", "response_mask"] - padded_tensors = {k: F.pad(data[k], pad=(max_prompt_len, 0), value=pad_token_id) for k in tensors_to_pad} - + if pad_to_prompt_len: + padded_tensors = {k: F.pad(data[k], pad=(max_prompt_len, 0), value=pad_token_id) for k in tensors_to_pad} + else: + padded_tensors = {k: data[k] for k in tensors_to_pad} extra_inputs = dict( prompts=data["prompts"], responses=data["responses"], @@ -198,7 +200,7 @@ def make_njt(data: TensorDict, tensor: Tensor) -> Tensor: return tensor def postprocess_log_prob_output(data: TensorDict, entropy: Tensor, log_probs: Tensor) -> TensorDict: - x_entropy = make_njt(data, entropy) + x_entropy = make_njt(data, entropy) x_log_probs = make_njt(data, log_probs) print(f"postprocess_log_prob_output: {x_entropy.shape=} {x_log_probs.shape=} {entropy.shape=} {log_probs.shape=}") @@ -212,8 +214,8 @@ def postprocess_log_prob_output(data: TensorDict, entropy: Tensor, log_probs: Te "log_probs": micro_log_probs[i], } output_lst.append({ - "model_output": model_output, - "metrics": {}, + "model_output": model_output, + "metrics": {}, "loss": 0.0, }) @@ -450,6 +452,8 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: def train_batch(self, data: TensorDict) -> TensorDict: assert self.loss_fn is not None, "loss function can't be None when calling train_batch" + use_zorro = True + # global_token_num should be a list of number of tokens of each seq in this batch global_token_num = tu.get(data, key="global_token_num") disable_auto_offload = tu.get(data, key="disable_auto_offload", default=False) @@ -519,10 +523,10 @@ def train_batch(self, data: TensorDict) -> TensorDict: position_ids=position_ids, attention_mask=data['attention_mask'], labels=input_ids, + use_zorro=use_zorro, ) print(f"{dss_batch_dict=}") - rollout_n = self.actor_config.rollout_n max_token_len_per_gpu = self.actor_config.ppo_max_token_len_per_gpu @@ -537,7 +541,7 @@ def safe_serialize(obj): #actor_config_as_dict = safe_serialize(self.actor_config) actor_config_as_dict = safe_serialize(actor_config_as_dict) - extra_inputs = prepare_extra_inputs(data, max_prompt_len) + extra_inputs = prepare_extra_inputs(data, max_prompt_len, pad_to_prompt_len=not use_zorro) extra_inputs["rollout_n"] = rollout_n extra_inputs["max_prompt_len"] = max_prompt_len extra_inputs["max_response_len"] = max_response_len @@ -756,15 +760,11 @@ def _update_config_params(self, data: TensorDict): if key not in data.keys(): tu.assign_non_tensor(data, **{key: val}) - + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="ref")) @DistProfiler.annotate(color="olive", role="ref_compute_log_prob") def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: - # return self._loaded_dump_data["full_ref_log_prob"] - # import pdb; pdb.set_trace() - # from verl.utils.tensordict_utils import chunk_tensordict - # batch = chunk_tensordict(data, 1) print(f"compute_ref_log_prob data: {data}") # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) @@ -791,14 +791,14 @@ def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: return final_output - + + + # TODO: Actor API Begin @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") def compute_log_prob(self, data: TensorDict) -> TensorDict: - # import pdb; pdb.set_trace() - # from verl.utils.tensordict_utils import chunk_tensordict - # batch = chunk_tensordict(data, 1) + print(f"compute_log_prob data: {data}") # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) @@ -807,7 +807,6 @@ def compute_log_prob(self, data: TensorDict) -> TensorDict: self._update_config_params(data) post_process_inputs = dict() # print(f"compute_log_prob: {post_process_inputs=}") - entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) # import pdb; pdb.set_trace() From 0c0d613cd58227f95e0f20899fcec6fbf6d4920b Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Tue, 7 Apr 2026 12:00:33 +0000 Subject: [PATCH 19/58] use_zorro env var; test zorro --- verl/trainer/ppo/arctic_rl_client.py | 14 ++++++++++++-- verl/workers/arctic_workers.py | 6 +++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index befd8c87822..ca0b883eaad 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -9,6 +9,7 @@ from verl.workers.rollout.replica import TokenOutput USE_ARCTIC_TRAINING_CLIENT = os.environ.get("USE_ARCTIC_TRAINING_CLIENT", "0") == "1" +USE_ARCTIC_ZORRO = os.environ.get("USE_ARCTIC_ZORRO", "0") == "1" def create_arctic_rl_client(config): @@ -21,7 +22,7 @@ def create_arctic_rl_client(config): placement_group=sched_pg, placement_group_capture_child_tasks=True, ), - )(cls).remote() + )(cls).remote(config) def create_meta_model(name_or_path: str): model_config = AutoConfig.from_pretrained(name_or_path) @@ -35,6 +36,7 @@ def __init__(self, config): config: verl's full config """ self.config = config + self.use_zorro = USE_ARCTIC_ZORRO #print(f"ArcticRLClient4VeRL {config=}") self.arctic_inference_client = DSSInferenceClient(dss_server_url="http://localhost:7000") @@ -42,6 +44,9 @@ def __init__(self, config): self.arctic_log_prob_client = DSSLogProbClient(dss_server_url="http://localhost:7000") + def is_zorro_enabled(self): + return self.use_zorro + def initialize(self, model_name: str): vllm_config = { "temperature": 0.0, @@ -162,9 +167,14 @@ class ArcticRLClientWrapper: Set USE_ARCTIC_TRAINING_CLIENT=1 env var to activate. """ - def __init__(self): + def __init__(self, config): + self.config = config self._client = None self.tokenizer = None + self.use_zorro = USE_ARCTIC_ZORRO + + def is_zorro_enabled(self): + return self.use_zorro def initialize(self, model_name: str): from arctic_training.arctic_rl import ArcticRLClient, ArcticRLClientConfig diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 8ad5a9b583b..e8079f88cfa 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -270,6 +270,7 @@ def __init__(self, config: TrainingWorkerConfig, actor_config: ActorConfig, arct self.optimizer_config = self.config.optimizer_config self.checkpoint_config = self.config.checkpoint_config self.device_name = get_device_name() + self.use_zorro = ray.get(self.arctic_rl_client.is_zorro_enabled.remote()) print(f"{self.engine_config=}") @@ -452,7 +453,6 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: def train_batch(self, data: TensorDict) -> TensorDict: assert self.loss_fn is not None, "loss function can't be None when calling train_batch" - use_zorro = True # global_token_num should be a list of number of tokens of each seq in this batch global_token_num = tu.get(data, key="global_token_num") @@ -523,7 +523,7 @@ def train_batch(self, data: TensorDict) -> TensorDict: position_ids=position_ids, attention_mask=data['attention_mask'], labels=input_ids, - use_zorro=use_zorro, + use_zorro=self.use_zorro, ) print(f"{dss_batch_dict=}") @@ -541,7 +541,7 @@ def safe_serialize(obj): #actor_config_as_dict = safe_serialize(self.actor_config) actor_config_as_dict = safe_serialize(actor_config_as_dict) - extra_inputs = prepare_extra_inputs(data, max_prompt_len, pad_to_prompt_len=not use_zorro) + extra_inputs = prepare_extra_inputs(data, max_prompt_len, pad_to_prompt_len=not self.use_zorro) extra_inputs["rollout_n"] = rollout_n extra_inputs["max_prompt_len"] = max_prompt_len extra_inputs["max_response_len"] = max_response_len From 8ec8c781eb4b3ae8c4ea4411f3cfe708a6e4a484 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 7 Apr 2026 20:44:50 +0000 Subject: [PATCH 20/58] zorro -> log_prob Signed-off-by: Stas Bekman --- verl/workers/arctic_workers.py | 142 +++++++++++++++++++++++++-------- 1 file changed, 109 insertions(+), 33 deletions(-) diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index e8079f88cfa..7aea2b3a251 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -165,7 +165,6 @@ def prepare_model_inputs_remove_padding(micro_batch: TensorDict): return model_inputs, output_args - def prepare_extra_inputs(data: TensorDict, max_prompt_len: int, pad_to_prompt_len=True) -> dict: pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) tensors_to_pad = ["advantages", "ref_log_prob", "old_log_probs", "response_mask"] @@ -189,6 +188,10 @@ def prepare_extra_inputs(data: TensorDict, max_prompt_len: int, pad_to_prompt_le return extra_inputs +def prepand_max_prompt_len_zeros(tensor: Tensor, max_prompt_len): + prepand = torch.zeros([tensor.shape[0], max_prompt_len], dtype=torch.int64, device=tensor.device) + return torch.cat([prepand, tensor], dim=1) + def make_njt(data: TensorDict, tensor: Tensor) -> Tensor: cu_seqlens = data["input_ids"].offsets() @@ -229,7 +232,7 @@ def prepare_padded_dss_batch_dict(data: TensorDict, pad_token_id) -> dict: from verl.workers.utils.padding import no_padding_2_padding_prompt_response orig_iput_ids_shape = input_ids.shape orig_position_ids_shape = position_ids.shape - input_ids, _, _ = no_padding_2_padding_prompt_response(tensor=input_ids, data=data, pad_token_id=pad_token_id) + input_ids, max_prompt_len, max_response_len = no_padding_2_padding_prompt_response(tensor=input_ids, data=data, pad_token_id=pad_token_id) # XXX: 0 pad on pos ids is odd, check the original - perhaps need to re-build pos ids? position_ids, _, _= no_padding_2_padding_prompt_response(tensor=position_ids, data=data, pad_token_id=0) attention_mask = data['attention_mask'] @@ -243,7 +246,7 @@ def prepare_padded_dss_batch_dict(data: TensorDict, pad_token_id) -> dict: labels=input_ids, ) - return dss_batch_dict + return dss_batch_dict, max_prompt_len, max_response_len class TrainingWorker(Worker, DistProfilerExtension): """ @@ -761,14 +764,11 @@ def _update_config_params(self, data: TensorDict): tu.assign_non_tensor(data, **{key: val}) - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="ref")) - @DistProfiler.annotate(color="olive", role="ref_compute_log_prob") - def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: + def compute_any_log_prob(self, data: TensorDict, compute_log_prob_fn) -> TensorDict: print(f"compute_ref_log_prob data: {data}") # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - dss_batch_dict = prepare_padded_dss_batch_dict(data, pad_token_id) + dss_batch_dict, max_prompt_len, max_response_len = prepare_padded_dss_batch_dict(data, pad_token_id) # print(f"{dss_batch_dict=}") # import pdb; pdb.set_trace() # self.dss_training_engine.forward(**dss_batch_dict) @@ -778,11 +778,35 @@ def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: # log_prob = self._loaded_dump_data["full_log_prob"] self._update_config_params(data) - post_process_inputs = dict() - entropy, log_probs = ray.get(self.arctic_rl_client.compute_ref_log_prob.remote(dss_batch_dict, post_process_inputs)) - batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) - model_output = batch_output.pop("model_output", {}) + rollout_n = self.actor_config.rollout_n + #max_token_len_per_gpu = self.actor_config.ppo_max_token_len_per_gpu + + extra_inputs = dict( + rollout_n=rollout_n, + max_prompt_len=max_prompt_len, + max_response_len=max_response_len, + max_token_len_per_gpu=data["max_token_len_per_gpu"], + ) + + post_process_inputs = dict(extra_inputs=extra_inputs) + entropy, log_probs = ray.get(compute_log_prob_fn.remote(dss_batch_dict, post_process_inputs)) + + print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") + + #batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) + #model_output = batch_output.pop("model_output", {}) + + # verl wants a full [bs, max_prompt_len+max_response_len] tensors and jagged + entropy = prepand_max_prompt_len_zeros(entropy, max_prompt_len) + log_probs = prepand_max_prompt_len_zeros(log_probs, max_prompt_len) + print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") + entropy = make_njt(data, entropy) + log_probs = make_njt(data, log_probs) + print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") + + model_output = dict(entropy=entropy, log_probs=log_probs) + metrics = { "mfu": 0.0, "loss": 1.0, @@ -792,33 +816,85 @@ def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: return final_output + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="ref")) + @DistProfiler.annotate(color="olive", role="ref_compute_log_prob") + def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: + return self.compute_any_log_prob(data, self.arctic_rl_client.compute_ref_log_prob) # TODO: Actor API Begin @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") def compute_log_prob(self, data: TensorDict) -> TensorDict: - - print(f"compute_log_prob data: {data}") - # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) - pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - dss_batch_dict = prepare_padded_dss_batch_dict(data, pad_token_id) - - self._update_config_params(data) - post_process_inputs = dict() - # print(f"compute_log_prob: {post_process_inputs=}") - entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) - - # import pdb; pdb.set_trace() - batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) - model_output = batch_output.pop("model_output", {}) - metrics = { - "mfu": 0.0, - "loss": 1.0, - } - final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) - - return final_output + return self.compute_any_log_prob(data, self.arctic_rl_client.compute_log_prob) + + + # @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="ref")) + # @DistProfiler.annotate(color="olive", role="ref_compute_log_prob") + # def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: + # print(f"compute_ref_log_prob data: {data}") + # # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + # pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) + # dss_batch_dict, max_prompt_len, max_response_len = prepare_padded_dss_batch_dict(data, pad_token_id) + # # print(f"{dss_batch_dict=}") + # # import pdb; pdb.set_trace() + # # self.dss_training_engine.forward(**dss_batch_dict) + # # loss = self.dss_training_engine.backward() + # # print(f"loss: {loss}") + # # import pdb; pdb.set_trace() + # # log_prob = self._loaded_dump_data["full_log_prob"] + + # self._update_config_params(data) + + # rollout_n = self.actor_config.rollout_n + # max_token_len_per_gpu = self.actor_config.ppo_max_token_len_per_gpu + + # extra_inputs = {} + # extra_inputs["rollout_n"] = rollout_n + # extra_inputs["max_prompt_len"] = max_prompt_len + # extra_inputs["max_response_len"] = max_response_len + + # post_process_inputs = dict(extra_inputs=extra_inputs) + # entropy, log_probs = ray.get(self.arctic_rl_client.compute_ref_log_prob.remote(dss_batch_dict, post_process_inputs)) + + # batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) + # model_output = batch_output.pop("model_output", {}) + # metrics = { + # "mfu": 0.0, + # "loss": 1.0, + # } + # final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) + + # return final_output + + + + + # # TODO: Actor API Begin + # @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + # @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") + # def compute_log_prob(self, data: TensorDict) -> TensorDict: + + # print(f"compute_log_prob data: {data}") + # # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) + # pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) + # dss_batch_dict = prepare_padded_dss_batch_dict(data, pad_token_id) + + # self._update_config_params(data) + # post_process_inputs = dict() + # # print(f"compute_log_prob: {post_process_inputs=}") + # entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) + + # # import pdb; pdb.set_trace() + # batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) + # model_output = batch_output.pop("model_output", {}) + # metrics = { + # "mfu": 0.0, + # "loss": 1.0, + # } + # final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) + + # return final_output @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) From 479e7d5becb6c4e3dbb2301483600a9fdb582e12 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Tue, 7 Apr 2026 20:55:06 +0000 Subject: [PATCH 21/58] WIP --- verl/trainer/ppo/arctic_rl_client.py | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index ca0b883eaad..76f9268a275 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -184,14 +184,14 @@ def initialize(self, model_name: str): port=7000, backend="local", # TODO: Grab GPU counts from VeRL config - training_gpus=1, + training_gpus=2, sample_gpus=1, log_prob_gpus=1, log_prob_engine="deepspeed", model_name=model_name, ds_config={ "train_micro_batch_size_per_gpu": 1, - "train_batch_size": 1, + "train_batch_size": 2, "gradient_accumulation_steps": 1, "sequence_parallel_size": 1, "zero_optimization": {"stage": 1}, @@ -213,6 +213,7 @@ def initialize(self, model_name: str): # so we need to set it manually. num_gpus = config.training_gpus + config.sample_gpus + config.log_prob_gpus os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in range(num_gpus)) + print(f"ArcticRLClientWrapper: {os.environ['CUDA_VISIBLE_DEVICES']=} {num_gpus=}") self._client = ArcticRLClient(config) self.tokenizer = AutoTokenizer.from_pretrained(model_name) @@ -249,29 +250,6 @@ def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict = Non if log_probs is not None: log_probs = torch.tensor(log_probs) - # TODO: AI fix to resolve problems after merge into arl branch. Need to verify correctness of this. - # The model returns packed (1, total_nnz) tensors, but downstream - # rm_padding expects padded (bsz, max_seq_len). Unpack and re-pad. - if post_process_inputs is not None: - cu_seqlens = post_process_inputs["cu_seqlens"] - seq_lengths = cu_seqlens.diff() - bsz = seq_lengths.shape[0] - max_seq_len = int(seq_lengths.max()) - - def _packed_to_padded(t): - flat = t.reshape(-1) - padded = torch.zeros(bsz, max_seq_len, dtype=flat.dtype, device=flat.device) - for i in range(bsz): - start = int(cu_seqlens[i]) - length = int(seq_lengths[i]) - padded[i, :length] = flat[start:start + length] - return padded - - if entropy is not None: - entropy = _packed_to_padded(entropy) - if log_probs is not None: - log_probs = _packed_to_padded(log_probs) - return entropy, log_probs def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): From f793988f9874f1e487425479ea5fb32f662a41e1 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Tue, 7 Apr 2026 14:17:10 -0700 Subject: [PATCH 22/58] ArcticTraining ARLClient integration (#9) --- verl/trainer/ppo/arctic_rl_client.py | 100 +++++++++++++++------------ 1 file changed, 55 insertions(+), 45 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index ca0b883eaad..a942347eca7 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -179,42 +179,75 @@ def is_zorro_enabled(self): def initialize(self, model_name: str): from arctic_training.arctic_rl import ArcticRLClient, ArcticRLClientConfig - config = ArcticRLClientConfig( + n_gpus = self.config.trainer.n_gpus_per_node + colocate = self.config.actor_rollout_ref.hybrid_engine + attn_implementation = self.config.actor_rollout_ref.model.override_config.get( + 'attn_implementation', 'eager' + ) + + actor_cfg = self.config.actor_rollout_ref.actor + optim_cfg = actor_cfg.optim + data_cfg = self.config.data + + micro_batch_size = actor_cfg.ppo_micro_batch_size_per_gpu or 1 + train_batch_size = data_cfg.train_batch_size + grad_accum_steps = max(1, train_batch_size // (micro_batch_size * n_gpus)) + seq_parallel_size = actor_cfg.fsdp_config.get("ulysses_sequence_parallel_size", 1) + max_length = data_cfg.max_prompt_length + data_cfg.max_response_length + + rollout_cfg = self.config.actor_rollout_ref.rollout + vllm_config = { + "tensor_parallel_size": rollout_cfg.tensor_model_parallel_size, + "gpu_memory_utilization": rollout_cfg.gpu_memory_utilization, + "max_model_len": rollout_cfg.get("max_model_len") or max_length, + "max_num_seqs": rollout_cfg.max_num_seqs, + "enforce_eager": rollout_cfg.enforce_eager, + "enable_chunked_prefill": rollout_cfg.enable_chunked_prefill, + } + if rollout_cfg.get("quantization"): + vllm_config["quantization"] = rollout_cfg.quantization + + rl_config = ArcticRLClientConfig( host="localhost", port=7000, backend="local", - # TODO: Grab GPU counts from VeRL config - training_gpus=1, - sample_gpus=1, - log_prob_gpus=1, + training_gpus=n_gpus, + sample_gpus=n_gpus, + log_prob_gpus=n_gpus, + colocate=colocate, log_prob_engine="deepspeed", model_name=model_name, ds_config={ - "train_micro_batch_size_per_gpu": 1, - "train_batch_size": 1, - "gradient_accumulation_steps": 1, - "sequence_parallel_size": 1, + "train_micro_batch_size_per_gpu": micro_batch_size, + "train_batch_size": train_batch_size, + "gradient_accumulation_steps": grad_accum_steps, + "sequence_parallel_size": seq_parallel_size, "zero_optimization": {"stage": 1}, }, - # TODO: Grab training config from VeRL config training_config={ - "optimizer": {"lr": 0.0002, "weight_decay": 0.0, "betas": [0.9, 0.999]}, - "lr_scheduler": {"warmup_ratio": 0.05}, - "training_horizon": 10, - "max_length": 8096, + "optimizer": { + "lr": optim_cfg.lr, + "weight_decay": optim_cfg.weight_decay, + "betas": list(optim_cfg.betas), + }, + "lr_scheduler": {"warmup_ratio": optim_cfg.lr_warmup_steps_ratio}, + "training_horizon": self.config.trainer.total_epochs, + "max_length": max_length, "model_config": None, - "attn_implementation": "eager", + "attn_implementation": attn_implementation, }, - vllm_config=None, + vllm_config=vllm_config, ) - # Feels like a hack, but ArcticRLClient is constructed as a ray remote - # actor with num_gpus=0 - This causes CUDA_VISIBLE_DEVICES to be empty, - # so we need to set it manually. - num_gpus = config.training_gpus + config.sample_gpus + config.log_prob_gpus - os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in range(num_gpus)) + # ArcticRLClient is constructed as a ray remote actor with num_gpus=0, + # which causes CUDA_VISIBLE_DEVICES to be empty. + if colocate: + num_visible = n_gpus + else: + num_visible = rl_config.training_gpus + rl_config.sample_gpus + rl_config.log_prob_gpus + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in range(num_visible)) - self._client = ArcticRLClient(config) + self._client = ArcticRLClient(rl_config) self.tokenizer = AutoTokenizer.from_pretrained(model_name) # TODO: Just for debugging - remove later @@ -249,29 +282,6 @@ def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict = Non if log_probs is not None: log_probs = torch.tensor(log_probs) - # TODO: AI fix to resolve problems after merge into arl branch. Need to verify correctness of this. - # The model returns packed (1, total_nnz) tensors, but downstream - # rm_padding expects padded (bsz, max_seq_len). Unpack and re-pad. - if post_process_inputs is not None: - cu_seqlens = post_process_inputs["cu_seqlens"] - seq_lengths = cu_seqlens.diff() - bsz = seq_lengths.shape[0] - max_seq_len = int(seq_lengths.max()) - - def _packed_to_padded(t): - flat = t.reshape(-1) - padded = torch.zeros(bsz, max_seq_len, dtype=flat.dtype, device=flat.device) - for i in range(bsz): - start = int(cu_seqlens[i]) - length = int(seq_lengths[i]) - padded[i, :length] = flat[start:start + length] - return padded - - if entropy is not None: - entropy = _packed_to_padded(entropy) - if log_probs is not None: - log_probs = _packed_to_padded(log_probs) - return entropy, log_probs def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): From bfae33fc8ad7cca4d52b6054809b12f4075966ca Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 7 Apr 2026 22:54:27 +0000 Subject: [PATCH 23/58] pass temp Signed-off-by: Stas Bekman --- verl/workers/arctic_workers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 7aea2b3a251..85470a7e604 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -545,9 +545,14 @@ def safe_serialize(obj): actor_config_as_dict = safe_serialize(actor_config_as_dict) extra_inputs = prepare_extra_inputs(data, max_prompt_len, pad_to_prompt_len=not self.use_zorro) - extra_inputs["rollout_n"] = rollout_n - extra_inputs["max_prompt_len"] = max_prompt_len - extra_inputs["max_response_len"] = max_response_len + extra_inputs.update( + rollout_n=rollout_n, + max_prompt_len=max_prompt_len, + max_response_len=max_response_len, + max_token_len_per_gpu=data["max_token_len_per_gpu"], + temperature=data["temperature"], + ) + policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) post_process_inputs = dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config, extra_inputs=extra_inputs) @@ -787,6 +792,7 @@ def compute_any_log_prob(self, data: TensorDict, compute_log_prob_fn) -> TensorD max_prompt_len=max_prompt_len, max_response_len=max_response_len, max_token_len_per_gpu=data["max_token_len_per_gpu"], + temperature=data["temperature"], ) post_process_inputs = dict(extra_inputs=extra_inputs) From 5a215b8397ce32a0e95c53ec9f6c75e051ed9a41 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 7 Apr 2026 22:55:12 +0000 Subject: [PATCH 24/58] helper Signed-off-by: Stas Bekman --- examples/arctic_rl/debug_at_gsm8k_grpo.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/arctic_rl/debug_at_gsm8k_grpo.sh b/examples/arctic_rl/debug_at_gsm8k_grpo.sh index e1c2fbf581f..2f6ca4d9a80 100755 --- a/examples/arctic_rl/debug_at_gsm8k_grpo.sh +++ b/examples/arctic_rl/debug_at_gsm8k_grpo.sh @@ -1,6 +1,8 @@ #!/bin/bash set -x +# enable if HF hub misbehaves/times out (assuming you have already cached the models locally) +# export HF_HUB_OFFLINE=1 export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 @@ -8,10 +10,12 @@ export RAY_DEDUP_LOGS=0 # we want to make sure this runs on non-gpu client export CUDA_VISIBLE_DEVICES= +# export USE_ARCTIC_ZORRO=1 + # BSZ=1024 -BSZ=1 -MBS=1 -UBS=1 +BSZ=2 +MBS=2 +UBS=2 ROLL_N=4 MAX_STEPS=4 # LR=0 @@ -50,7 +54,7 @@ python3 -m verl.trainer.main_ppo \ data.val_files=/code/shared/gsm8k/test.parquet \ data.train_batch_size=${BSZ} \ data.max_prompt_length=64 \ - data.max_response_length=96 \ + data.max_response_length=16 \ data.filter_overlong_prompts=True \ data.truncation='error' \ data.shuffle=False \ From 27e3ada8542b9e5a4066d2c074769f6cc1535f7f Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Wed, 8 Apr 2026 03:39:45 +0000 Subject: [PATCH 25/58] Remove utility --- verl/utils/fsdp_utils.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/verl/utils/fsdp_utils.py b/verl/utils/fsdp_utils.py index 11fc247d441..8bca54fa88c 100644 --- a/verl/utils/fsdp_utils.py +++ b/verl/utils/fsdp_utils.py @@ -227,19 +227,6 @@ def load_fsdp_optimizer(optimizer, device_id): state[key] = value.to(device_id, non_blocking=True) -@torch.no_grad() -def get_fsdp_optimizer_devices(optimizer) -> list[torch.device]: - devices = set() - for param_group in optimizer.param_groups: - for param in param_group["params"]: - state = optimizer.state[param] - devices.add(param.device) - for key, value in state.items(): - if isinstance(value, torch.Tensor): - devices.add(value.device) - - return list(devices) - @contextmanager def meta_device_init(): """ From dd9719e64274e1398e84077a8b839b397921ecba Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Wed, 8 Apr 2026 19:59:00 +0000 Subject: [PATCH 26/58] Cleanup --- examples/arctic_rl/debug_at_gsm8k_grpo.sh | 2 +- examples/arctic_rl/run_arctic_gsm8k_grpo.sh | 3 + verl/trainer/ppo/arctic_rl_client.py | 4 +- verl/workers/arctic_workers.py | 170 ++++---------------- 4 files changed, 33 insertions(+), 146 deletions(-) diff --git a/examples/arctic_rl/debug_at_gsm8k_grpo.sh b/examples/arctic_rl/debug_at_gsm8k_grpo.sh index 2f6ca4d9a80..d2fd56e9a53 100755 --- a/examples/arctic_rl/debug_at_gsm8k_grpo.sh +++ b/examples/arctic_rl/debug_at_gsm8k_grpo.sh @@ -2,7 +2,7 @@ set -x # enable if HF hub misbehaves/times out (assuming you have already cached the models locally) -# export HF_HUB_OFFLINE=1 +export HF_HUB_OFFLINE=1 export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh index 43487db3b93..83f5793c48a 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh @@ -5,6 +5,9 @@ set -x export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 +export HF_HUB_OFFLINE=1 +# export USE_ARCTIC_ZORRO=1 + # we want to make sure this runs on non-gpu client export CUDA_VISIBLE_DEVICES= diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index a942347eca7..8a9f4315b8b 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -143,11 +143,11 @@ def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): dss_batch_dict.update(post_process_inputs=post_process_inputs) - #_ = self.training_engine.forward(**dss_batch_dict, post_process_inputs=post_process_inputs) _ = self.training_engine.forward(**dss_batch_dict) loss, metrics = self.training_engine.backward() - self.training_engine.step() + global_steps, last_lr = self.training_engine.step() + metrics.update({"global_steps": [global_steps], "last_lr": [last_lr]}) print(f"arctic_rl_client.update_actor: {loss=}") print(f"arctic_rl_client.update_actor: {metrics=}") return loss.cpu().item(), metrics diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 85470a7e604..1455fb15252 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -165,23 +165,17 @@ def prepare_model_inputs_remove_padding(micro_batch: TensorDict): return model_inputs, output_args -def prepare_extra_inputs(data: TensorDict, max_prompt_len: int, pad_to_prompt_len=True) -> dict: - pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - tensors_to_pad = ["advantages", "ref_log_prob", "old_log_probs", "response_mask"] - if pad_to_prompt_len: - padded_tensors = {k: F.pad(data[k], pad=(max_prompt_len, 0), value=pad_token_id) for k in tensors_to_pad} - else: - padded_tensors = {k: data[k] for k in tensors_to_pad} +def prepare_extra_inputs(data: TensorDict) -> dict: extra_inputs = dict( prompts=data["prompts"], responses=data["responses"], attention_mask=data["attention_mask"], max_token_len_per_gpu=data["max_token_len_per_gpu"], global_batch_size=data["global_batch_size"], - response_mask=padded_tensors["response_mask"], - old_log_probs=padded_tensors["old_log_probs"], - advantages=padded_tensors["advantages"], - ref_log_prob=padded_tensors["ref_log_prob"], + response_mask=data["response_mask"], + old_log_probs=data["old_log_probs"], + advantages=data["advantages"], + ref_log_prob=data["ref_log_prob"], rollout_is_weights=data.get("rollout_is_weights", None), batch_num_tokens=data["loss_mask"].sum(), ) @@ -202,28 +196,6 @@ def make_njt(data: TensorDict, tensor: Tensor) -> Tensor: tensor = torch.nested.nested_tensor_from_jagged(tensor, cu_seqlens) return tensor -def postprocess_log_prob_output(data: TensorDict, entropy: Tensor, log_probs: Tensor) -> TensorDict: - x_entropy = make_njt(data, entropy) - x_log_probs = make_njt(data, log_probs) - - print(f"postprocess_log_prob_output: {x_entropy.shape=} {x_log_probs.shape=} {entropy.shape=} {log_probs.shape=}") - - micro_entropy = [t.unsqueeze(0) for t in x_entropy.unbind()] - micro_log_probs = [t.unsqueeze(0) for t in x_log_probs.unbind()] - output_lst = [] - for i in range(len(micro_entropy)): - model_output = { - "entropy": micro_entropy[i], - "log_probs": micro_log_probs[i], - } - output_lst.append({ - "model_output": model_output, - "metrics": {}, - "loss": 0.0, - }) - - return postprocess_batch_func(output_lst=output_lst, indices=None, data=data) - def prepare_padded_dss_batch_dict(data: TensorDict, pad_token_id) -> dict: input_ids = data['input_ids'] @@ -544,13 +516,26 @@ def safe_serialize(obj): #actor_config_as_dict = safe_serialize(self.actor_config) actor_config_as_dict = safe_serialize(actor_config_as_dict) - extra_inputs = prepare_extra_inputs(data, max_prompt_len, pad_to_prompt_len=not self.use_zorro) - extra_inputs.update( + # TODO: move to init since globally constant + extra_inputs = dict( rollout_n=rollout_n, max_prompt_len=max_prompt_len, max_response_len=max_response_len, max_token_len_per_gpu=data["max_token_len_per_gpu"], - temperature=data["temperature"], + temperature=data["temperature"], + ) + + extra_inputs.update( + prompts=data["prompts"], + responses=data["responses"], + attention_mask=data["attention_mask"], + global_batch_size=data["global_batch_size"], + response_mask=data["response_mask"], + old_log_probs=data["old_log_probs"], + advantages=data["advantages"], + ref_log_prob=data["ref_log_prob"], + rollout_is_weights=data.get("rollout_is_weights", None), + batch_num_tokens=data["loss_mask"].sum(), ) policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) @@ -574,13 +559,13 @@ def safe_serialize(obj): loss, metrics = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) # output = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) # print(f"update_actor: {loss=}") - # print(f"update_actor: {metrics=}") + print(f"update_actor: {metrics=}") from verl.utils.metric import AggregationType, Metric # XXX: fix me - we need to aggregate the metrics metrics = {k:Metric(value=v[0], aggregation=AggregationType.MEAN) for k,v in metrics.items()} - + metrics["lr"] = metrics.pop("last_lr") delta_time = timer.last # XXX: fix me @@ -596,10 +581,7 @@ def safe_serialize(obj): # print(f"{data=}") print(f"{data["input_ids"].shape=}") - model_output = { - # XXX: fix me - made a copy of existing same shape tensor for now - # 'log_probs': batch[0]["ref_log_prob"] - } + model_output = {} # expected output so far # @@ -627,21 +609,6 @@ def safe_serialize(obj): loss=loss, ) - update_lr_scheduler = tu.get(data, key="update_lr_scheduler", default=False) - # XXX: fix me - update_lr_scheduler = False - # update lr scheduler - if update_lr_scheduler: - lr = self.engine.lr_scheduler_step() - else: - lr = None - - - # we don't need model_output in training. Maybe we change out mind later - #output.pop("model_output") - if lr is not None: - output["metrics"]["lr"] = lr - final_output = self._postprocess_output( output, global_token_num=global_token_num, @@ -718,18 +685,6 @@ def __init__(self, config: DictConfig, role: str, **kwargs): self.set_dispatch_collect(mesh_name="actor", **self.actor.get_dispatch_collect()) - # from verl.workers.actor import DataParallelPPOActor - - # # hacks to appease to DataParallelPPOActor - # import torch.distributed - # torch.distributed.get_rank = lambda: 0 - - # actor_cfg = omega_conf_to_dataclass(self.config.actor) - # self.actor = DataParallelPPOActor( - # # XXX: hijack actor_module - # config=actor_cfg, actor_module=None, actor_optimizer=None - # ) - @register(dispatch_mode=Dispatch.ONE_TO_ALL) def init_model(self): @@ -828,99 +783,32 @@ def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: return self.compute_any_log_prob(data, self.arctic_rl_client.compute_ref_log_prob) - # TODO: Actor API Begin @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") def compute_log_prob(self, data: TensorDict) -> TensorDict: return self.compute_any_log_prob(data, self.arctic_rl_client.compute_log_prob) - # @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="ref")) - # @DistProfiler.annotate(color="olive", role="ref_compute_log_prob") - # def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: - # print(f"compute_ref_log_prob data: {data}") - # # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) - # pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - # dss_batch_dict, max_prompt_len, max_response_len = prepare_padded_dss_batch_dict(data, pad_token_id) - # # print(f"{dss_batch_dict=}") - # # import pdb; pdb.set_trace() - # # self.dss_training_engine.forward(**dss_batch_dict) - # # loss = self.dss_training_engine.backward() - # # print(f"loss: {loss}") - # # import pdb; pdb.set_trace() - # # log_prob = self._loaded_dump_data["full_log_prob"] - - # self._update_config_params(data) - - # rollout_n = self.actor_config.rollout_n - # max_token_len_per_gpu = self.actor_config.ppo_max_token_len_per_gpu - - # extra_inputs = {} - # extra_inputs["rollout_n"] = rollout_n - # extra_inputs["max_prompt_len"] = max_prompt_len - # extra_inputs["max_response_len"] = max_response_len - - # post_process_inputs = dict(extra_inputs=extra_inputs) - # entropy, log_probs = ray.get(self.arctic_rl_client.compute_ref_log_prob.remote(dss_batch_dict, post_process_inputs)) - - # batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) - # model_output = batch_output.pop("model_output", {}) - # metrics = { - # "mfu": 0.0, - # "loss": 1.0, - # } - # final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) - - # return final_output - - - - - # # TODO: Actor API Begin - # @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) - # @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") - # def compute_log_prob(self, data: TensorDict) -> TensorDict: - - # print(f"compute_log_prob data: {data}") - # # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) - # pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - # dss_batch_dict = prepare_padded_dss_batch_dict(data, pad_token_id) - - # self._update_config_params(data) - # post_process_inputs = dict() - # # print(f"compute_log_prob: {post_process_inputs=}") - # entropy, log_probs = ray.get(self.arctic_rl_client.compute_log_prob.remote(dss_batch_dict, post_process_inputs)) - - # # import pdb; pdb.set_trace() - # batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) - # model_output = batch_output.pop("model_output", {}) - # metrics = { - # "mfu": 0.0, - # "loss": 1.0, - # } - # final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) - - # return final_output - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) @DistProfiler.annotate(color="red", role="actor_update") def update_actor(self, data: TensorDict) -> TensorDict: output = self.actor.train_global_batch(data=data) return output.cpu() if output is not None else None - + # TODO: Load Checkpoint API @register(dispatch_mode=Dispatch.ONE_TO_ALL) def load_checkpoint(self, local_path, hdfs_path=None, del_local_after_load=False): assert "actor" in self.role, "load_checkpoint only support actor role" return + # TODO: Save Checkpoint API @register(dispatch_mode=Dispatch.ONE_TO_ALL) def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): assert "actor" in self.role, "save_checkpoint only support actor role" return + # TODO: Update Weights API @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) async def update_weights(self, global_steps: int = None): """Update weights from trainer to rollout. @@ -932,10 +820,7 @@ async def update_weights(self, global_steps: int = None): """ return - # TODO: Actor API End - - # TODO: Rollout API Begin @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="rollout")) async def generate_sequences(self, batch: DataProto): # print(f"{batch.non_tensor_batch=}") @@ -955,7 +840,6 @@ async def generate_sequences(self, batch: DataProto): return gen_batch_output # return self._loaded_dump_data["gen_batch_output"] - # TODO: Rollout API End # TODO: CheckpointManager API Begin @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) From b88967a0b85883037b116ba19f6da3b125b4ff5c Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 9 Apr 2026 18:28:38 +0000 Subject: [PATCH 27/58] removing train batch nesting Signed-off-by: Stas Bekman --- verl/workers/arctic_workers.py | 110 ++++++++++++--------------------- 1 file changed, 39 insertions(+), 71 deletions(-) diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 1455fb15252..f58a4b19bd4 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -385,49 +385,20 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: Returns: """ + assert self.loss_fn is not None, "loss function can't be None when calling train_global_batch" + disable_auto_offload = tu.pop(data, key="disable_auto_offload", default=False) self.engine_config = self.config.engine_config - with ( - Timer(name="train_batch", logger=None), - ): - # update - global_token_num = data["input_ids"].offsets().diff().tolist() # (total_nnz,) - tu.assign_non_tensor( - data, - global_token_num=NonTensorData(global_token_num), - update_lr_scheduler=True, - disable_auto_offload=disable_auto_offload, - ) - - actor_output = self.train_batch(data) - - output_metrics = tu.get(actor_output, "metrics") - - metrics = {} - for key, val in output_metrics.items(): - # print(f"metrics {key=} {val=}") - - # flattn dp and micro batch - if isinstance(val, list): - output_metrics[key] = ( - Metric.aggregate_dp(val) - if isinstance(val[0], Metric) - else list(chain.from_iterable(val)) - ) - - append_to_dict(metrics, output_metrics) - - output = tu.get_tensordict(tensor_dict={}, non_tensor_dict={"metrics": metrics}).cpu() - - return output - - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train"), blocking=False) - def train_batch(self, data: TensorDict) -> TensorDict: - assert self.loss_fn is not None, "loss function can't be None when calling train_batch" - + # update + global_token_num = data["input_ids"].offsets().diff().tolist() # (total_nnz,) + tu.assign_non_tensor( + data, + global_token_num=NonTensorData(global_token_num), + update_lr_scheduler=True, + disable_auto_offload=disable_auto_offload, + ) # global_token_num should be a list of number of tokens of each seq in this batch global_token_num = tu.get(data, key="global_token_num") @@ -503,26 +474,13 @@ def train_batch(self, data: TensorDict) -> TensorDict: print(f"{dss_batch_dict=}") rollout_n = self.actor_config.rollout_n - max_token_len_per_gpu = self.actor_config.ppo_max_token_len_per_gpu - - # we need to serialize the config object to dict - # dataclasses.asdict only returns keys that are defined at init (vars will do more) - but perhaps we want `asdict`? - actor_config_as_dict = vars(self.actor_config) - print(f"update_actor: {self.actor_config=}") - print(f"update_actor: {actor_config_as_dict=}") - import json - def safe_serialize(obj): - return json.loads(json.dumps(obj, default=lambda o: None)) - #actor_config_as_dict = safe_serialize(self.actor_config) - actor_config_as_dict = safe_serialize(actor_config_as_dict) - # TODO: move to init since globally constant extra_inputs = dict( rollout_n=rollout_n, max_prompt_len=max_prompt_len, max_response_len=max_response_len, max_token_len_per_gpu=data["max_token_len_per_gpu"], - temperature=data["temperature"], + temperature=data["temperature"], ) extra_inputs.update( @@ -538,24 +496,21 @@ def safe_serialize(obj): batch_num_tokens=data["loss_mask"].sum(), ) + # we need to serialize the config object to dict + # dataclasses.asdict only returns keys that are defined at init (vars will do more) - but perhaps we want `asdict`? + actor_config_as_dict = vars(self.actor_config) + print(f"update_actor: {self.actor_config=}") + print(f"update_actor: {actor_config_as_dict=}") + import json + def safe_serialize(obj): + return json.loads(json.dumps(obj, default=lambda o: None)) + actor_config_as_dict = safe_serialize(actor_config_as_dict) + policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) post_process_inputs = dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config, extra_inputs=extra_inputs) # print(f"update_actor: {post_process_inputs=}") - # XXX: pass the original batch as post_process_inputs["batch"] - the ppo loss function expects data["prompts"] - # it got stripped and is not in dss_batch_dict -# File "/code/users/stas/github/sf/dss-platform/dss/processors/verl.py", line 90, in fwd_post_process_ppo_loss -# return ppo_loss(config, model_output, data) -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -# File "/code/users/stas/github/sf/dss-platform/dss/processors/verl.py", line 100, in ppo_loss -# log_prob = no_padding_2_padding(model_output["log_probs"], data) -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -# File "/code/users/stas/github/sf/arctic-verl/verl/workers/utils/padding.py", line 99, in no_padding_2_padding -# prompt_ids = data["prompts"] -# ~~~~^^^^^^^^^^^ -# KeyError: 'prompts' - loss, metrics = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) # output = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) # print(f"update_actor: {loss=}") @@ -581,7 +536,6 @@ def safe_serialize(obj): # print(f"{data=}") print(f"{data["input_ids"].shape=}") - model_output = {} # expected output so far # @@ -601,15 +555,14 @@ def safe_serialize(obj): # } # } - - #output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict=non_tensor_dict) + model_output = {} output = dict( model_output=model_output, metrics=metrics, loss=loss, ) - final_output = self._postprocess_output( + actor_output = self._postprocess_output( output, global_token_num=global_token_num, delta_time=delta_time, @@ -617,10 +570,25 @@ def safe_serialize(obj): images_seqlens=images_seqlens, ).cpu() - return final_output + output_metrics = tu.get(actor_output, "metrics") + metrics = {} + for key, val in output_metrics.items(): + # print(f"metrics {key=} {val=}") + + # flattn dp and micro batch + if isinstance(val, list): + output_metrics[key] = ( + Metric.aggregate_dp(val) + if isinstance(val[0], Metric) + else list(chain.from_iterable(val)) + ) + append_to_dict(metrics, output_metrics) + output = tu.get_tensordict(tensor_dict={}, non_tensor_dict={"metrics": metrics}).cpu() + + return output From aa3fc1b4ef521b48c5a369e3a314979dbbafb40c Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 9 Apr 2026 19:31:02 +0000 Subject: [PATCH 28/58] ref_log_prob optional --- verl/workers/arctic_workers.py | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 1455fb15252..991ea884235 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -533,11 +533,13 @@ def safe_serialize(obj): response_mask=data["response_mask"], old_log_probs=data["old_log_probs"], advantages=data["advantages"], - ref_log_prob=data["ref_log_prob"], rollout_is_weights=data.get("rollout_is_weights", None), batch_num_tokens=data["loss_mask"].sum(), ) + if self.actor_config.use_kl_loss: + extra_inputs["ref_log_prob"] = data["ref_log_prob"] + policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) post_process_inputs = dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config, extra_inputs=extra_inputs) @@ -820,27 +822,6 @@ async def update_weights(self, global_steps: int = None): """ return - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="rollout")) - async def generate_sequences(self, batch: DataProto): - # print(f"{batch.non_tensor_batch=}") - raw_prompts = list(batch.non_tensor_batch["raw_prompt"]) - # print(f"{raw_prompts=}") - prompts = self.tokenizer.apply_chat_template( - raw_prompts, - add_generation_prompt=True, - tokenize=False, - ) - # import pdb; pdb.set_trace() - # print(f"prompts: {prompts}") - metrics = {} - - gen_batch_output = self.arctic_inference_engine.generate(prompts=prompts) - - return gen_batch_output - # return self._loaded_dump_data["gen_batch_output"] - - # TODO: CheckpointManager API Begin @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) async def sleep_replicas(self): From 810b6f5098c5bda9458e92e9fe580fcb9c02172e Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Thu, 9 Apr 2026 17:19:30 -0700 Subject: [PATCH 29/58] fix for ArcticTraining-dss API update (#15) --- verl/trainer/ppo/arctic_rl_client.py | 41 +++++++++++++++++++--------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 8a9f4315b8b..f3e8c92aedc 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -180,6 +180,7 @@ def initialize(self, model_name: str): from arctic_training.arctic_rl import ArcticRLClient, ArcticRLClientConfig n_gpus = self.config.trainer.n_gpus_per_node + #n_gpus = 2 colocate = self.config.actor_rollout_ref.hybrid_engine attn_implementation = self.config.actor_rollout_ref.model.override_config.get( 'attn_implementation', 'eager' @@ -267,28 +268,40 @@ def compute_ref_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict = return self.compute_log_prob(dss_batch_dict, post_process_inputs) def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict = None): + print(f"[ArcticRLWrapper] compute_log_prob INPUT: " + f"{{{', '.join(f'{k}: {v.shape}' for k, v in dss_batch_dict.items() if isinstance(v, torch.Tensor))}}}") batch = { "kwargs": dss_batch_dict, - "context": {"labels": dss_batch_dict["labels"]}, + "context": {"input_ids": dss_batch_dict["input_ids"]}, + "processing": { + "post": ["compute_logprobs"], + "loss_fn": None, + }, } - result = self._client.fwd_no_grad(batch, post_processors=["entropy_logprobs"]) + result = self._client.fwd_no_grad(batch) outputs = result.get("model_outputs", result) - entropy = outputs.get("entropy") - log_probs = outputs.get("log_probs") - - if entropy is not None: - entropy = torch.tensor(entropy) - if log_probs is not None: + log_probs = outputs.get("logprobs") + if log_probs is not None and not isinstance(log_probs, torch.Tensor): log_probs = torch.tensor(log_probs) + # The pipeline doesn't return true entropy; approximate as + # -logprobs to match what grpo_loss uses internally (grpo.py:249). + entropy = -log_probs if log_probs is not None else None + + print(f"[ArcticRLWrapper] compute_log_prob OUTPUT: " + f"entropy={entropy.shape if entropy is not None else None} " + f"log_probs={log_probs.shape if log_probs is not None else None}") return entropy, log_probs def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): - # TODO: Does this align with the ArcticRLClient4VeRL + dss-platform:verl_integration branch? extra = post_process_inputs.get("extra_inputs", {}) seq_len = dss_batch_dict["input_ids"].shape[-1] + print(f"[ArcticRLWrapper] update_actor INPUT: " + f"{{{', '.join(f'{k}: {v.shape}' for k, v in dss_batch_dict.items() if isinstance(v, torch.Tensor))}}} " + f"| extra: {{{', '.join(f'{k}: {v.shape}' for k, v in extra.items() if isinstance(v, torch.Tensor))}}}") + def _left_pad(t: torch.Tensor) -> torch.Tensor: """Left-pad a response-only tensor to full sequence length with zeros.""" pad_len = seq_len - t.shape[-1] @@ -298,21 +311,23 @@ def _left_pad(t: torch.Tensor) -> torch.Tensor: return torch.cat([pad, t], dim=-1) context = { - "labels": dss_batch_dict["labels"], - "old_logprobs": _left_pad(extra["old_log_probs"]), + "input_ids": dss_batch_dict["input_ids"], + "old_log_probs_shifted": _left_pad(extra["old_log_probs"]), "advantages": _left_pad(extra["advantages"]), "loss_mask": _left_pad(extra["response_mask"]), } + print(f"[ArcticRLWrapper] update_actor CONTEXT: " + f"{{{', '.join(f'{k}: {v.shape}' for k, v in context.items() if isinstance(v, torch.Tensor))}}}") batch = {"kwargs": dss_batch_dict, "context": context} - result = self._client.fwd_bwd(batch, loss_fn="grpo") + result = self._client.fwd_bwd(batch, processing={"loss_fn": "grpo", "post": ["compute_logprobs"]}) self._client.step() loss = result.get("avg_loss", 0.0) raw_metrics = result.get("post_process_outputs", {}) - # Caller expects metrics values to be lists (does v[0]) metrics = {k: v if isinstance(v, list) else [v] for k, v in raw_metrics.items()} + print(f"[ArcticRLWrapper] update_actor OUTPUT: loss={loss} metrics={metrics}") return loss, metrics def destroy(self): From 9c1296af2880cb1a5ea4e9cf8527fa9e061c7ab6 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Fri, 10 Apr 2026 02:46:17 +0000 Subject: [PATCH 30/58] Cleaning up --- verl/trainer/ppo/arctic_rl_client.py | 57 +++++++++++----------- verl/workers/arctic_workers.py | 72 ++++++++++++---------------- 2 files changed, 60 insertions(+), 69 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 8a9f4315b8b..f9821b90305 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -30,6 +30,7 @@ def create_meta_model(name_or_path: str): meta_model = AutoModelForCausalLM.from_config(model_config) return meta_model + class ArcticRLClient4VeRL: def __init__(self, config): """ @@ -109,48 +110,48 @@ def generate(self, prompt_ids, sampling_params) -> TokenOutput: sampling_params=sampling_params, ) + # TODO: this should use the reference engine instead of the training engine - def compute_ref_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict): - dss_batch_dict.update(post_process_inputs=post_process_inputs) - entropy, log_probs = self.training_engine.fwd_no_grad(**dss_batch_dict) - if entropy is not None: - entropy = torch.tensor(entropy).squeeze() - if log_probs is not None: - log_probs = torch.tensor(log_probs).squeeze() - print(f"arctic_rl_client.compute_ref_log_prob: {entropy.shape=}, {log_probs.shape=}") - return entropy, log_probs + def compute_ref_log_prob(self, payload: dict): + response = self.training_engine.fwd_no_grad(**payload) + # if entropy is not None: + # entropy = torch.tensor(entropy).squeeze() + # if log_probs is not None: + # log_probs = torch.tensor(log_probs).squeeze() + print(f"arctic_rl_client.compute_ref_log_prob: {response['batch']['entropy'].shape=}, {response['batch']['log_probs'].shape=}") + return response - def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict): - dss_batch_dict.update(post_process_inputs=post_process_inputs) - + def compute_log_prob(self, payload: dict): # XXX: somehow we need to differentiate which model is this called on ref vs actor - at the moment it's always actor hardcoded - entropy, log_probs = self.training_engine.fwd_no_grad(**dss_batch_dict) + response = self.training_engine.fwd_no_grad(**payload) # XXX: for some reason no_padding_2_padding expects a 1D tensor - not sure how it'll work for # bs>1 # I think it may have to do with tensor.is_nested - different path/logic # so most likely we need to convert these 2 into TensorDict - if entropy is not None: - entropy = torch.tensor(entropy).squeeze() - if log_probs is not None: - log_probs = torch.tensor(log_probs).squeeze() - print(f"arctic_rl_client.compute_log_prob: {entropy.shape=}, {log_probs.shape=}") - return entropy, log_probs + # if entropy is not None: + # entropy = torch.tensor(entropy).squeeze() + # if log_probs is not None: + # log_probs = torch.tensor(log_probs).squeeze() + print(f"arctic_rl_client.compute_log_prob: {response['batch']['entropy'].shape=}, {response['batch']['log_probs'].shape=}") + return response - def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): + def update_actor(self, payload: dict): + _ = self.training_engine.forward(**payload) + bwd_response = self.training_engine.backward() + step_response = self.training_engine.step() + + step_response["metrics"].update(**bwd_response["metrics"]) - dss_batch_dict.update(post_process_inputs=post_process_inputs) + # metrics.update({"global_steps": [global_steps], "last_lr": [last_lr]}) - _ = self.training_engine.forward(**dss_batch_dict) - loss, metrics = self.training_engine.backward() - global_steps, last_lr = self.training_engine.step() + # print(f"arctic_rl_client.update_actor: {loss=}") + # print(f"arctic_rl_client.update_actor: {metrics=}") + # return loss.cpu().item(), metrics - metrics.update({"global_steps": [global_steps], "last_lr": [last_lr]}) - print(f"arctic_rl_client.update_actor: {loss=}") - print(f"arctic_rl_client.update_actor: {metrics=}") - return loss.cpu().item(), metrics + return step_response def destroy(self): self.training_engine.destroy() diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 7bba319eed9..3a045f6e593 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -431,7 +431,6 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: print(f"update_actor data: {data}") # XXX: fix me - padding_token = 100 input_ids = data['input_ids'] position_ids = data['position_ids'] #input_ids = input_ids.unbind() @@ -464,39 +463,35 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: #print(f"{output_args=}") #import pdb; pdb.set_trace() - dss_batch_dict = dict( + batch = dict( input_ids=input_ids, position_ids=position_ids, attention_mask=data['attention_mask'], labels=input_ids, - use_zorro=self.use_zorro, + prompts=data["prompts"], + responses=data["responses"], + response_mask=data["response_mask"], + old_log_probs=data["old_log_probs"], + advantages=data["advantages"], ) - print(f"{dss_batch_dict=}") + if self.actor_config.use_kl_loss: + batch["ref_log_prob"] = data["ref_log_prob"] + + print(f"{batch=}") - rollout_n = self.actor_config.rollout_n # TODO: move to init since globally constant - extra_inputs = dict( - rollout_n=rollout_n, + meta = dict( + rollout_n=self.actor_config.rollout_n, max_prompt_len=max_prompt_len, max_response_len=max_response_len, max_token_len_per_gpu=data["max_token_len_per_gpu"], temperature=data["temperature"], - ) - - extra_inputs.update( - prompts=data["prompts"], - responses=data["responses"], - attention_mask=data["attention_mask"], + use_zorro=self.use_zorro, global_batch_size=data["global_batch_size"], - response_mask=data["response_mask"], - old_log_probs=data["old_log_probs"], - advantages=data["advantages"], rollout_is_weights=data.get("rollout_is_weights", None), batch_num_tokens=data["loss_mask"].sum(), ) - if self.actor_config.use_kl_loss: - extra_inputs["ref_log_prob"] = data["ref_log_prob"] # we need to serialize the config object to dict # dataclasses.asdict only returns keys that are defined at init (vars will do more) - but perhaps we want `asdict`? actor_config_as_dict = vars(self.actor_config) @@ -509,12 +504,16 @@ def safe_serialize(obj): policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) - post_process_inputs = dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config, extra_inputs=extra_inputs) + meta.update(dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config)) # print(f"update_actor: {post_process_inputs=}") - loss, metrics = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) + + payload = dict(batch=batch, meta=meta) + response = ray.get(self.arctic_rl_client.update_actor.remote(payload)) # output = ray.get(self.arctic_rl_client.update_actor.remote(dss_batch_dict, post_process_inputs)) # print(f"update_actor: {loss=}") + metrics = response['metrics'] + loss = metrics.pop("loss") print(f"update_actor: {metrics=}") @@ -695,52 +694,43 @@ def _update_config_params(self, data: TensorDict): def compute_any_log_prob(self, data: TensorDict, compute_log_prob_fn) -> TensorDict: print(f"compute_ref_log_prob data: {data}") - # dss_batch_dict, output_args = prepare_model_inputs_remove_padding(data) pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - dss_batch_dict, max_prompt_len, max_response_len = prepare_padded_dss_batch_dict(data, pad_token_id) - # print(f"{dss_batch_dict=}") - # import pdb; pdb.set_trace() - # self.dss_training_engine.forward(**dss_batch_dict) - # loss = self.dss_training_engine.backward() - # print(f"loss: {loss}") - # import pdb; pdb.set_trace() - # log_prob = self._loaded_dump_data["full_log_prob"] + batch, max_prompt_len, max_response_len = prepare_padded_dss_batch_dict(data, pad_token_id) self._update_config_params(data) - rollout_n = self.actor_config.rollout_n #max_token_len_per_gpu = self.actor_config.ppo_max_token_len_per_gpu - extra_inputs = dict( - rollout_n=rollout_n, + meta = dict( + rollout_n=self.actor_config.rollout_n, max_prompt_len=max_prompt_len, max_response_len=max_response_len, max_token_len_per_gpu=data["max_token_len_per_gpu"], temperature=data["temperature"], ) - post_process_inputs = dict(extra_inputs=extra_inputs) - entropy, log_probs = ray.get(compute_log_prob_fn.remote(dss_batch_dict, post_process_inputs)) + payload = dict(batch=batch, meta=meta) - print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") + response = ray.get(compute_log_prob_fn.remote(payload)) + + print(f"compute_any_log_prob: {response['batch']['entropy'].shape=} {response['batch']['log_probs'].shape=}") #batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) #model_output = batch_output.pop("model_output", {}) # verl wants a full [bs, max_prompt_len+max_response_len] tensors and jagged - entropy = prepand_max_prompt_len_zeros(entropy, max_prompt_len) - log_probs = prepand_max_prompt_len_zeros(log_probs, max_prompt_len) + entropy = prepand_max_prompt_len_zeros(response['batch']['entropy'], max_prompt_len) + log_probs = prepand_max_prompt_len_zeros(response['batch']['log_probs'], max_prompt_len) print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") entropy = make_njt(data, entropy) log_probs = make_njt(data, log_probs) print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") model_output = dict(entropy=entropy, log_probs=log_probs) + metrics = response['metrics'] + # TODO: fix me - mfu is not computed here + metrics["mfu"] = 0.0 - metrics = { - "mfu": 0.0, - "loss": 1.0, - } final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) return final_output From 44393b5941b85c7b89d0eca8293fa739083b8e87 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Fri, 10 Apr 2026 17:24:21 +0000 Subject: [PATCH 31/58] dataloader seed --- examples/arctic_rl/run_arctic_gsm8k_grpo.sh | 6 ++++-- examples/arctic_rl/run_gsm8k_grpo.sh | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh index 83f5793c48a..7b6d1aa2059 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh @@ -19,8 +19,8 @@ ROLL_N=2 MAX_STEPS=4 # LR=0 LR=1e-6 -LOGGER=console -# LOGGER="['console','wandb']" +# LOGGER=console +LOGGER="['console','wandb']" USE_KL_LOSS=True # USE_KL_LOSS=False # REMOVE_PADDING=True @@ -58,6 +58,8 @@ python3 -m verl.trainer.main_ppo \ data.filter_overlong_prompts=True \ data.truncation='error' \ data.shuffle=False \ + +data.seed=42 \ + +actor_rollout_ref.actor.data_loader_seed=42 \ reward.num_workers=1 \ actor_rollout_ref.rollout.agent.num_workers=1 \ actor_rollout_ref.model.path=${MODEL} \ diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh index 1317d8091d9..a59371ed9f5 100755 --- a/examples/arctic_rl/run_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_gsm8k_grpo.sh @@ -13,8 +13,8 @@ ROLL_N=2 MAX_STEPS=4 # LR=0 LR=1e-6 -LOGGER=console -# LOGGER="['console','wandb']" +# LOGGER=console +LOGGER="['console','wandb']" USE_KL_LOSS=True # USE_KL_LOSS=False # REMOVE_PADDING=True @@ -40,6 +40,8 @@ python3 -m verl.trainer.main_ppo \ data.filter_overlong_prompts=True \ data.truncation='error' \ data.shuffle=False \ + +data.seed=42 \ + +actor_rollout_ref.actor.data_loader_seed=42 \ reward.num_workers=1 \ actor_rollout_ref.rollout.agent.num_workers=1 \ actor_rollout_ref.model.path=${MODEL} \ From bd779d226017f9de2e30c89392ffbcc365e1d3f1 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Fri, 10 Apr 2026 17:59:30 +0000 Subject: [PATCH 32/58] W&B --- examples/arctic_rl/run_arctic_gsm8k_grpo.sh | 10 +++++----- examples/arctic_rl/run_gsm8k_grpo.sh | 8 ++++---- verl/workers/arctic_workers.py | 17 ++++++++++------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh index 7b6d1aa2059..0f772a3afc7 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo.sh @@ -6,7 +6,7 @@ export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 export HF_HUB_OFFLINE=1 -# export USE_ARCTIC_ZORRO=1 +export USE_ARCTIC_ZORRO=1 # we want to make sure this runs on non-gpu client export CUDA_VISIBLE_DEVICES= @@ -17,8 +17,8 @@ MBS=2 UBS=2 ROLL_N=2 MAX_STEPS=4 -# LR=0 -LR=1e-6 +LR=0 +# LR=1e-6 # LOGGER=console LOGGER="['console','wandb']" USE_KL_LOSS=True @@ -35,7 +35,7 @@ NGPU_PER_NODE=1 ROLLOUT_NAME=arctic # entry point into ArcticRL USE_ARCTIC_RL=True -experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_at${USE_ARCTIC_RL}" +experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_zorro${USE_ARCTIC_ZORRO}" gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) if [[ $gpu_name == *"H200"* ]]; then @@ -59,7 +59,7 @@ python3 -m verl.trainer.main_ppo \ data.truncation='error' \ data.shuffle=False \ +data.seed=42 \ - +actor_rollout_ref.actor.data_loader_seed=42 \ + actor_rollout_ref.actor.data_loader_seed=42 \ reward.num_workers=1 \ actor_rollout_ref.rollout.agent.num_workers=1 \ actor_rollout_ref.model.path=${MODEL} \ diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh index a59371ed9f5..bb9abcc2046 100755 --- a/examples/arctic_rl/run_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_gsm8k_grpo.sh @@ -11,8 +11,8 @@ MBS=2 UBS=2 ROLL_N=2 MAX_STEPS=4 -# LR=0 -LR=1e-6 +LR=0 +# LR=1e-6 # LOGGER=console LOGGER="['console','wandb']" USE_KL_LOSS=True @@ -28,7 +28,7 @@ USE_LEGACY_WORKER_IMPL=disable NGPU_PER_NODE=1 ROLLOUT_NAME=vllm -experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}" +experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_baseline" python3 -m verl.trainer.main_ppo \ algorithm.adv_estimator=grpo \ @@ -41,7 +41,7 @@ python3 -m verl.trainer.main_ppo \ data.truncation='error' \ data.shuffle=False \ +data.seed=42 \ - +actor_rollout_ref.actor.data_loader_seed=42 \ + actor_rollout_ref.actor.data_loader_seed=42 \ reward.num_workers=1 \ actor_rollout_ref.rollout.agent.num_workers=1 \ actor_rollout_ref.model.path=${MODEL} \ diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 3a045f6e593..052efa3e33d 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -12,6 +12,7 @@ from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer from deepspeed.utils import OnDevice from verl.utils import tensordict_utils as tu +from verl.utils import hf_tokenizer import os import ray from verl.utils.config import omega_conf_to_dataclass @@ -239,6 +240,7 @@ def __init__(self, config: TrainingWorkerConfig, actor_config: ActorConfig, arct self.arctic_rl_client = arctic_rl_client self.tokenizer = tokenizer + self.pad_token_id = self.tokenizer.pad_token_id self.model_config = self.config.model_config self.engine_config = self.config.engine_config @@ -437,10 +439,9 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: from verl.workers.utils.padding import no_padding_2_padding_prompt_response # XXX: move to init - if self.tokenizer.pad_token_id is None: - self.tokenizer.pad_token_id = self.tokenizer.eos_token_id + - input_ids, max_prompt_len, max_response_len = no_padding_2_padding_prompt_response(tensor=input_ids, data=data, pad_token_id=self.tokenizer.pad_token_id) + input_ids, max_prompt_len, max_response_len = no_padding_2_padding_prompt_response(tensor=input_ids, data=data, pad_token_id=self.pad_token_id) # XXX: 0 pad on pos ids is odd, check the original - perhaps need to re-build pos ids? position_ids, _, _= no_padding_2_padding_prompt_response(tensor=position_ids, data=data, pad_token_id=0) print(f"{input_ids.shape=}") @@ -643,8 +644,11 @@ def __init__(self, config: DictConfig, role: str, **kwargs): assert self.config.rollout.log_prob_micro_batch_size_per_gpu is not None assert self.config.actor.ppo_micro_batch_size_per_gpu is not None - # XXX: fix me - duplicated in _init_engines and model hardcoded - self.tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") + trust_remote_code=self.config.model.get("trust_remote_code", False) + self.tokenizer = hf_tokenizer(self.config.model.path, trust_remote_code=trust_remote_code) + if self.tokenizer.pad_token_id is None: + self.tokenizer.pad_token_id = self.tokenizer.eos_token_id + self.pad_token_id = self.tokenizer.pad_token_id self.actor = TrainingWorker(config=actor_training_config, actor_config=actor_config, arctic_rl_client=self.arctic_rl_client, tokenizer=self.tokenizer ) self.actor.reset() @@ -694,8 +698,7 @@ def _update_config_params(self, data: TensorDict): def compute_any_log_prob(self, data: TensorDict, compute_log_prob_fn) -> TensorDict: print(f"compute_ref_log_prob data: {data}") - pad_token_id = tu.get_non_tensor_data(data=data, key="pad_token_id", default=0) - batch, max_prompt_len, max_response_len = prepare_padded_dss_batch_dict(data, pad_token_id) + batch, max_prompt_len, max_response_len = prepare_padded_dss_batch_dict(data, self.pad_token_id) self._update_config_params(data) From 2a4403f191810139a0e5282fd9b787bf56958414 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 10 Apr 2026 21:06:54 +0000 Subject: [PATCH 33/58] split scripts Signed-off-by: Stas Bekman --- examples/arctic_rl/debug_at_gsm8k_grpo.sh | 2 +- .../run_arctic_gsm8k_grpo_zorro_no.sh | 103 ++++++++++++++++++ ....sh => run_arctic_gsm8k_grpo_zorro_yes.sh} | 2 +- 3 files changed, 105 insertions(+), 2 deletions(-) create mode 100755 examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh rename examples/arctic_rl/{run_arctic_gsm8k_grpo.sh => run_arctic_gsm8k_grpo_zorro_yes.sh} (99%) diff --git a/examples/arctic_rl/debug_at_gsm8k_grpo.sh b/examples/arctic_rl/debug_at_gsm8k_grpo.sh index d2fd56e9a53..4845ff784a6 100755 --- a/examples/arctic_rl/debug_at_gsm8k_grpo.sh +++ b/examples/arctic_rl/debug_at_gsm8k_grpo.sh @@ -10,7 +10,7 @@ export RAY_DEDUP_LOGS=0 # we want to make sure this runs on non-gpu client export CUDA_VISIBLE_DEVICES= -# export USE_ARCTIC_ZORRO=1 +export USE_ARCTIC_ZORRO=1 # BSZ=1024 BSZ=2 diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh new file mode 100755 index 00000000000..d4bd95a7cce --- /dev/null +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +export HF_HUB_OFFLINE=1 +export USE_ARCTIC_ZORRO=0 + +# we want to make sure this runs on non-gpu client +export CUDA_VISIBLE_DEVICES= + +# BSZ=1024 +BSZ=2 +MBS=2 +UBS=2 +ROLL_N=2 +MAX_STEPS=4 +LR=0 +# LR=1e-6 +# LOGGER=console +LOGGER="['console','wandb']" +USE_KL_LOSS=True +# USE_KL_LOSS=False +# REMOVE_PADDING=True +REMOVE_PADDING=False +MODEL="Qwen/Qwen3-0.6B" +# STRATEGY="fsdp" +STRATEGY="fsdp2" +PYTHONUNBUFFERED=1 +HYDRA_FULL_ERROR=1 +USE_LEGACY_WORKER_IMPL=disable +NGPU_PER_NODE=1 +ROLLOUT_NAME=arctic # entry point into ArcticRL +USE_ARCTIC_RL=True + +experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_zorro${USE_ARCTIC_ZORRO}" + +gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) +if [[ $gpu_name == *"H200"* ]]; then + echo "Running on Hopper" + flash_attention_v=flash_attention_3 +elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then + echo "Running on Blackwell" + flash_attention_v=flash_attention_2 +else + echo "Running on unknown: $gpu_name; don't know which FA version to use" +fi + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=/code/shared/gsm8k/train.parquet \ + data.val_files=/code/shared/gsm8k/test.parquet \ + data.train_batch_size=${BSZ} \ + data.max_prompt_length=64 \ + data.max_response_length=512 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + +data.seed=42 \ + actor_rollout_ref.actor.data_loader_seed=42 \ + reward.num_workers=1 \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.model.path=${MODEL} \ + actor_rollout_ref.actor.optim.lr=${LR} \ + actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ + actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ + actor_rollout_ref.actor.strategy=${STRATEGY} \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.n=${ROLL_N} \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.ref.fsdp_config.param_offload=False \ + actor_rollout_ref.ref.strategy=${STRATEGY} \ + algorithm.use_kl_in_reward=False \ + trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ + trainer.use_arctic_rl=${USE_ARCTIC_RL} \ + trainer.critic_warmup=0 \ + trainer.logger=${LOGGER} \ + trainer.experiment_name=${experiment_name} \ + trainer.project_name='verl_arctic_grpo_gsm8k' \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_training_steps=${MAX_STEPS} \ + trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log + + # trainer.total_training_steps=${MAX_STEPS} \ diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh similarity index 99% rename from examples/arctic_rl/run_arctic_gsm8k_grpo.sh rename to examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh index 0f772a3afc7..794d8adfe5f 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh @@ -54,7 +54,7 @@ python3 -m verl.trainer.main_ppo \ data.val_files=/code/shared/gsm8k/test.parquet \ data.train_batch_size=${BSZ} \ data.max_prompt_length=64 \ - data.max_response_length=96 \ + data.max_response_length=512 \ data.filter_overlong_prompts=True \ data.truncation='error' \ data.shuffle=False \ From d6f6aba5dd51dd0b6dd8d77874d60165aef86942 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Fri, 10 Apr 2026 23:02:38 +0000 Subject: [PATCH 34/58] text2sql recipe --- examples/arctic_rl/bird_reward.py | 272 ++++++++++++++++++ ...un_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh | 135 +++++++++ ...n_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh | 135 +++++++++ examples/arctic_rl/run_gsm8k_grpo.sh | 8 +- .../run_qwen3_1.7b_bird_grpo_baseline.sh | 132 +++++++++ 5 files changed, 678 insertions(+), 4 deletions(-) create mode 100644 examples/arctic_rl/bird_reward.py create mode 100755 examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh create mode 100755 examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh create mode 100755 examples/arctic_rl/run_qwen3_1.7b_bird_grpo_baseline.sh diff --git a/examples/arctic_rl/bird_reward.py b/examples/arctic_rl/bird_reward.py new file mode 100644 index 00000000000..92a0bec4a30 --- /dev/null +++ b/examples/arctic_rl/bird_reward.py @@ -0,0 +1,272 @@ +""" +SQL reward function for BIRD RL training, adapted from SnowflakeDialectSQLRewardManagerV6b. + +Uses SQLite execution instead of Snowflake. Compatible with verl's +custom_reward_function mechanism via compute_score(). + +Reward scheme (matching V6b non-semantic-model behavior): + 1.0 - Predicted SQL produces the same result set as gold SQL + 0.1 - Predicted SQL executes successfully but produces wrong results, + OR SQL was extracted successfully (format bonus) + 0.0 - No SQL extracted, SQL fails to execute, or timeout +""" + +import json +import re +import sqlite3 +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError +from functools import lru_cache + +SQL_TIMEOUT = 30 +DEFAULT_LIMIT_NUMBER = 5000 +FORMAT_REWARD_BONUS = 0.1 + + +# --------------------------------------------------------------------------- +# SQL extraction (mirrors V6b's extract_solution / _extract_sql_omnisql) +# --------------------------------------------------------------------------- + +def _extract_sql_omnisql(message: str) -> str: + """Extract SQL from ```sql ... ``` markdown blocks (last valid block).""" + pattern = r"```sql\s*(.*?)\s*```" + sql_blocks = re.findall(pattern, message, re.DOTALL) + for block in reversed(sql_blocks): + if len(block.strip()) > 6: + return block.strip() + return "" + + +def _extract_sql_generic_block(message: str) -> str: + """Extract SQL from generic ``` ... ``` blocks containing SELECT.""" + blocks = re.findall(r"```\s*(.*?)\s*```", message, re.DOTALL) + for block in reversed(blocks): + if "SELECT" in block.upper() and len(block.strip()) > 6: + return block.strip() + return "" + + +def _extract_sql_analyst(message: str) -> str: + """Extract SQL from ```json { "sql": "..." } ``` blocks.""" + block = re.search(r"```\s*json(.*?)```", message, re.DOTALL) + if block is None: + return "" + json_str = block.group(1) + idx_left = json_str.rfind("{") + idx_right = json_str.find("}") + if idx_left == -1 or idx_right == -1: + return "" + json_str = json_str[idx_left : idx_right + 1] + try: + return json.loads(json_str.replace("\\n", "\n").replace("\\'", "'"), strict=False).get("sql", "") + except Exception: + return "" + + +def _extract_sql_raw_select(message: str) -> str: + """Fallback: extract a raw SELECT statement.""" + match = re.search(r"(SELECT\s+.+?)(?:\n\n|$)", message, re.DOTALL | re.IGNORECASE) + if match: + return match.group(1).strip() + return "" + + +def extract_sql(response: str) -> str: + """Extract SQL from model response, following V6b's extraction pipeline. + + 1. Split on to isolate the answer portion + 2. Try ```sql blocks + 3. Try generic ``` blocks with SELECT + 4. Try ```json blocks with {"sql": ...} + 5. Fallback to raw SELECT statement + """ + if "" in response: + answer_part = response.split("", 1)[1] + else: + answer_part = response + + sql = _extract_sql_omnisql(answer_part) + if sql: + return sql + + sql = _extract_sql_generic_block(answer_part) + if sql: + return sql + + sql = _extract_sql_analyst(answer_part) + if sql: + return sql + + return _extract_sql_raw_select(answer_part) + + +# --------------------------------------------------------------------------- +# Format validation (mirrors V6b's validate_response_structure) +# --------------------------------------------------------------------------- + +def validate_response_format(response: str) -> bool: + """Check that the response has exactly one ... pair, properly nested.""" + start_positions = [m.start() for m in re.finditer(r"", response)] + end_positions = [m.start() for m in re.finditer(r"", response)] + + if len(start_positions) != 1 or len(end_positions) != 1: + return False + return start_positions[0] < end_positions[0] + + +# --------------------------------------------------------------------------- +# LIMIT addition (mirrors V6b's _add_limit_to_query) +# --------------------------------------------------------------------------- + +def _add_limit_to_query(query: str, limit: int = DEFAULT_LIMIT_NUMBER) -> str: + """Add LIMIT clause if the query doesn't already have one.""" + if not query: + return query + upper = query.upper() + if "LIMIT " in upper or "LIMIT\n" in upper or "LIMIT\t" in upper: + return query + return query.rstrip().rstrip(";").rstrip() + f" LIMIT {limit};" + + +# --------------------------------------------------------------------------- +# SQLite execution and comparison +# --------------------------------------------------------------------------- + +def _execute_sql(db_path: str, sql: str, timeout: float = SQL_TIMEOUT) -> frozenset | Exception: + """Execute SQL against a SQLite database and return result as frozenset of row tuples.""" + try: + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5) + deadline = __import__("time").monotonic() + timeout + def _check_cancel(): + if __import__("time").monotonic() > deadline: + return 1 + return 0 + conn.set_progress_handler(_check_cancel, 1000) + cursor = conn.cursor() + cursor.execute(sql) + rows = cursor.fetchall() + conn.close() + return frozenset(rows) + except sqlite3.OperationalError as e: + if "interrupt" in str(e).lower(): + return TimeoutError(f"SQL execution exceeded {timeout}s") + return e + except Exception as e: + return e + + +def _execute_with_timeout(db_path: str, sql: str, timeout: float = SQL_TIMEOUT) -> frozenset | Exception: + """Execute SQL with a timeout using a thread pool. + + Uses shutdown(wait=False) to avoid blocking if the SQLite thread is stuck. + The SQLite progress handler provides cooperative cancellation. + """ + executor = ThreadPoolExecutor(max_workers=1) + future = executor.submit(_execute_sql, db_path, sql, timeout) + try: + return future.result(timeout=timeout + 2) + except FuturesTimeoutError: + return TimeoutError(f"SQL execution exceeded {timeout}s") + except Exception as e: + return e + finally: + executor.shutdown(wait=False) + + +def _compare_results( + db_path: str, + pred_sql: str, + gold_sqls: list[str], + timeout: float = SQL_TIMEOUT, +) -> tuple[float, bool]: + """Execute and compare predicted SQL against all gold SQLs. + + Returns (reward, execution_success) matching V6b's non-semantic-model logic: + - 1.0 if pred result == any gold result (frozenset match) + - 0.1 if pred executes but doesn't match any gold + - 0.0 if pred fails to execute + + Caches gold results within this call to avoid re-execution. + """ + pred_sql_limited = _add_limit_to_query(pred_sql) + pred_result = _execute_with_timeout(db_path, pred_sql_limited, timeout) + + if isinstance(pred_result, Exception): + return 0.0, False + + gold_cache: dict[str, frozenset | Exception] = {} + scores = [] + for gold_sql in gold_sqls: + if gold_sql not in gold_cache: + gold_sql_limited = _add_limit_to_query(gold_sql) + gold_cache[gold_sql] = _execute_with_timeout(db_path, gold_sql_limited, timeout) + gold_result = gold_cache[gold_sql] + + if isinstance(gold_result, Exception): + scores.append(0.0) + continue + + if pred_result == gold_result: + scores.append(1.0) + else: + scores.append(0.1) + + return (max(scores) if scores else 0.0), True + + +# --------------------------------------------------------------------------- +# Main entry point +# --------------------------------------------------------------------------- + +def compute_score(data_source, solution_str, ground_truth, extra_info=None, **kwargs): + """Compute reward score for SQL generation (verl custom_reward_function interface). + + Mirrors SnowflakeDialectSQLRewardManagerV6b logic with SQLite execution: + 1. Extract SQL from model response + 2. Validate response format ( tags) + 3. Execute predicted and gold SQL against SQLite + 4. Compare results (frozenset match) + 5. Apply format bonus + + Args: + data_source: Dataset identifier (e.g. "bird") + solution_str: Full model response text (decoded) + ground_truth: Gold SQL query string + extra_info: Dict with at minimum {"db_path": "/path/to/db.sqlite"}. + Optionally {"alternative_answers": [...]} for multiple gold SQLs. + + Returns: + dict with "score" (float), "format_correct" (float), "execution_success" (float) + """ + if extra_info is None: + return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} + + db_path = extra_info.get("db_path", "") + if not db_path: + return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} + + pred_sql = extract_sql(solution_str) + format_correct = float(bool(pred_sql) and validate_response_format(solution_str)) + + if not pred_sql: + return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} + + alternative_answers = extra_info.get("alternative_answers") + if alternative_answers and len(alternative_answers) > 0: + gold_sqls = [str(s).strip() for s in alternative_answers if s and str(s).strip()] + else: + gold_sqls = [ground_truth] if ground_truth else [] + + if not gold_sqls: + return {"score": 0.0, "format_correct": format_correct, "execution_success": 0.0} + + reward, execution_success = _compare_results(db_path, pred_sql, gold_sqls) + + if format_correct: + reward = max(reward, FORMAT_REWARD_BONUS) + + return { + "score": reward, + "format_correct": format_correct, + "execution_success": float(execution_success), + } diff --git a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh new file mode 100755 index 00000000000..9cc1bfa02b9 --- /dev/null +++ b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# GRPO training for Qwen3-1.7B on BIRD SQL dataset +# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) +# +# 1 node, 8 GPUs +# +# Prerequisites: +# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py +# 2. pip install func_timeout + +set -x + +experiment_name='qwen3_1.7b_bird_grpo_zorro_no' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH}" +MAX_STEPS=4 +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +export HF_HUB_OFFLINE=1 + +export CUDA_VISIBLE_DEVICES= +USE_ARCTIC_RL=True # entry point into ArcticRL + +USE_LEGACY_WORKER_IMPL=disable +ROLLOUT_NAME=arctic +NUM_AGENT_WORKERS=1 +NGPU_PER_NODE=1 + +# BSZ=128 +# PROMPT_LEN=16384 +# RESPONSE_LEN=4096 +# ROLL_N=16 + +BSZ=2 +PROMPT_LEN=16384 +RESPONSE_LEN=4096 +ROLL_N=2 +# LOGGER=console +LOGGER="['console','wandb']" + +gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) +if [[ $gpu_name == *"H200"* ]]; then + echo "Running on Hopper" + flash_attention_v=flash_attention_3 +elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then + echo "Running on Blackwell" + flash_attention_v=flash_attention_2 +else + echo "Running on unknown: $gpu_name; don't know which FA version to use" +fi + +# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" +# TRAIN_FILES="${DATA_DIR}/train.parquet" +# VAL_FILES="${DATA_DIR}/val.parquet" + +DATA_DIR="/code/shared/open-source-text2sql" +TRAIN_FILES="${DATA_DIR}/train.parquet" +VAL_FILES="${DATA_DIR}/val.parquet" + + +# LOG_PROBS=True +LOG_PROBS=False + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + algorithm.norm_adv_by_std_in_grpo=True \ + algorithm.use_kl_in_reward=False \ + algorithm.kl_ctrl.kl_coef=0.001 \ + data.train_files=${TRAIN_FILES} \ + data.val_files=${VAL_FILES} \ + data.train_batch_size=${BSZ} \ + data.max_prompt_length=${PROMPT_LEN} \ + data.max_response_length=${RESPONSE_LEN} \ + data.filter_overlong_prompts=True \ + data.filter_overlong_prompts_workers=1 \ + data.truncation=left \ + actor_rollout_ref.model.path=Qwen/Qwen3-1.7B \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ + actor_rollout_ref.model.use_liger=True \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.actor.use_torch_compile=True \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${BSZ} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.lr=5e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ + actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=${ROLL_N} \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.top_p=1.0 \ + actor_rollout_ref.rollout.calculate_log_probs=${LOG_PROBS} \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.max_num_seqs=256 \ + actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ + actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ + actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ + actor_rollout_ref.rollout.val_kwargs.temperature=0 \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.nccl_timeout=1800 \ + trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ + trainer.use_arctic_rl=${USE_ARCTIC_RL} \ + trainer.balance_batch=False \ + trainer.default_local_dir=/data-fast/sql-rl/${experiment_name} \ + trainer.logger=${LOGGER} \ + trainer.project_name=arctic_rl_bird_sql \ + trainer.experiment_name=${experiment_name} \ + trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=10 \ + trainer.val_before_train=False \ + custom_reward_function.path="${SCRIPT_DIR}/bird_reward.py" \ + custom_reward_function.name=compute_score \ + trainer.total_training_steps=${MAX_STEPS} \ + "$@" 2>&1 | tee ${experiment_name}.log diff --git a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh new file mode 100755 index 00000000000..b1382390ccb --- /dev/null +++ b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# GRPO training for Qwen3-1.7B on BIRD SQL dataset +# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) +# +# 1 node, 8 GPUs +# +# Prerequisites: +# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py +# 2. pip install func_timeout + +set -x + +experiment_name='qwen3_1.7b_bird_grpo_zorro_yes' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH}" +MAX_STEPS=4 +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +export HF_HUB_OFFLINE=1 +export USE_ARCTIC_ZORRO=1 + +export CUDA_VISIBLE_DEVICES= +USE_ARCTIC_RL=True # entry point into ArcticRL + +USE_LEGACY_WORKER_IMPL=disable +ROLLOUT_NAME=arctic +NUM_AGENT_WORKERS=1 +NGPU_PER_NODE=1 + +# BSZ=128 +# PROMPT_LEN=16384 +# RESPONSE_LEN=4096 +# ROLL_N=16 + +BSZ=2 +PROMPT_LEN=16384 +RESPONSE_LEN=4096 +ROLL_N=2 +# LOGGER=console +LOGGER="['console','wandb']" + +gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) +if [[ $gpu_name == *"H200"* ]]; then + echo "Running on Hopper" + flash_attention_v=flash_attention_3 +elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then + echo "Running on Blackwell" + flash_attention_v=flash_attention_2 +else + echo "Running on unknown: $gpu_name; don't know which FA version to use" +fi + +# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" +# TRAIN_FILES="${DATA_DIR}/train.parquet" +# VAL_FILES="${DATA_DIR}/val.parquet" + +DATA_DIR="/code/shared/open-source-text2sql" +TRAIN_FILES="${DATA_DIR}/train.parquet" +VAL_FILES="${DATA_DIR}/val.parquet" + +# LOG_PROBS=True +LOG_PROBS=False + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + algorithm.norm_adv_by_std_in_grpo=True \ + algorithm.use_kl_in_reward=False \ + algorithm.kl_ctrl.kl_coef=0.001 \ + data.train_files=${TRAIN_FILES} \ + data.val_files=${VAL_FILES} \ + data.train_batch_size=${BSZ} \ + data.max_prompt_length=${PROMPT_LEN} \ + data.max_response_length=${RESPONSE_LEN} \ + data.filter_overlong_prompts=True \ + data.filter_overlong_prompts_workers=1 \ + data.truncation=left \ + actor_rollout_ref.model.path=Qwen/Qwen3-1.7B \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ + actor_rollout_ref.model.use_liger=True \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.actor.use_torch_compile=True \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${BSZ} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.lr=5e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ + actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=${ROLL_N} \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.top_p=1.0 \ + actor_rollout_ref.rollout.calculate_log_probs=${LOG_PROBS} \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.max_num_seqs=256 \ + actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ + actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ + actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ + actor_rollout_ref.rollout.val_kwargs.temperature=0 \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.nccl_timeout=1800 \ + trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ + trainer.use_arctic_rl=${USE_ARCTIC_RL} \ + trainer.balance_batch=False \ + trainer.default_local_dir=/data-fast/sql-rl/${experiment_name} \ + trainer.logger=${LOGGER} \ + trainer.project_name=arctic_rl_bird_sql \ + trainer.experiment_name=${experiment_name} \ + trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=10 \ + trainer.val_before_train=False \ + custom_reward_function.path="${SCRIPT_DIR}/bird_reward.py" \ + custom_reward_function.name=compute_score \ + trainer.total_training_steps=${MAX_STEPS} \ + "$@" 2>&1 | tee ${experiment_name}.log diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh index bb9abcc2046..2328483f738 100755 --- a/examples/arctic_rl/run_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_gsm8k_grpo.sh @@ -11,10 +11,10 @@ MBS=2 UBS=2 ROLL_N=2 MAX_STEPS=4 -LR=0 -# LR=1e-6 -# LOGGER=console -LOGGER="['console','wandb']" +# LR=0 +LR=1e-6 +LOGGER=console +# LOGGER="['console','wandb']" USE_KL_LOSS=True # USE_KL_LOSS=False # REMOVE_PADDING=True diff --git a/examples/arctic_rl/run_qwen3_1.7b_bird_grpo_baseline.sh b/examples/arctic_rl/run_qwen3_1.7b_bird_grpo_baseline.sh new file mode 100755 index 00000000000..a555ffdf523 --- /dev/null +++ b/examples/arctic_rl/run_qwen3_1.7b_bird_grpo_baseline.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# GRPO training for Qwen3-1.7B on BIRD SQL dataset +# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) +# +# 1 node, 8 GPUs +# +# Prerequisites: +# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py +# 2. pip install func_timeout + +set -x + +experiment_name='qwen3_1.7b_bird_grpo_baseline' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH}" +MAX_STEPS=4 +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +export HF_HUB_OFFLINE=1 +USE_LEGACY_WORKER_IMPL=disable +ROLLOUT_NAME=vllm +NUM_AGENT_WORKERS=1 +NGPU_PER_NODE=1 + +# BSZ=128 +# PROMPT_LEN=16384 +# RESPONSE_LEN=4096 +# ROLL_N=16 + +BSZ=2 +PROMPT_LEN=16384 +RESPONSE_LEN=4096 +ROLL_N=2 + +# LOGGER=console +LOGGER="['console','wandb']" + +gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) +if [[ $gpu_name == *"H200"* ]]; then + echo "Running on Hopper" + flash_attention_v=flash_attention_3 +elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then + echo "Running on Blackwell" + flash_attention_v=flash_attention_2 +else + echo "Running on unknown: $gpu_name; don't know which FA version to use" +fi + +# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" +# TRAIN_FILES="${DATA_DIR}/train.parquet" +# VAL_FILES="${DATA_DIR}/val.parquet" + + +DATA_DIR="/code/shared/open-source-text2sql" +TRAIN_FILES="${DATA_DIR}/train.parquet" +VAL_FILES="${DATA_DIR}/val.parquet" + + +# LOG_PROBS=True +LOG_PROBS=False + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + algorithm.norm_adv_by_std_in_grpo=True \ + algorithm.use_kl_in_reward=False \ + algorithm.kl_ctrl.kl_coef=0.001 \ + data.train_files=${TRAIN_FILES} \ + data.val_files=${VAL_FILES} \ + data.train_batch_size=${BSZ} \ + data.max_prompt_length=${PROMPT_LEN} \ + data.max_response_length=${RESPONSE_LEN} \ + data.filter_overlong_prompts=True \ + data.filter_overlong_prompts_workers=1 \ + data.truncation=left \ + actor_rollout_ref.model.path=Qwen/Qwen3-1.7B \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ + actor_rollout_ref.model.use_liger=True \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.actor.use_torch_compile=True \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=${BSZ} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.lr=5e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ + actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=${ROLL_N} \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.top_p=1.0 \ + actor_rollout_ref.rollout.calculate_log_probs=${LOG_PROBS} \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.max_num_seqs=256 \ + actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ + actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ + actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ + actor_rollout_ref.rollout.val_kwargs.temperature=0 \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.nccl_timeout=1800 \ + trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ + trainer.balance_batch=False \ + trainer.default_local_dir=/data-fast/sql-rl/${experiment_name} \ + trainer.logger=${LOGGER} \ + trainer.project_name=arctic_rl_bird_sql \ + trainer.experiment_name=${experiment_name} \ + trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=10 \ + trainer.val_before_train=False \ + custom_reward_function.path="${SCRIPT_DIR}/bird_reward.py" \ + custom_reward_function.name=compute_score \ + trainer.total_training_steps=${MAX_STEPS} \ + "$@" 2>&1 | tee ${experiment_name}.log From 670626d9d2c7d3b8f3b94beeb5d564730fe2a916 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Tue, 14 Apr 2026 20:44:42 +0000 Subject: [PATCH 35/58] ARL integration --- .../run_arctic_gsm8k_grpo_zorro_no.sh | 35 ++++-- verl/trainer/config/ppo_trainer.yaml | 11 ++ verl/trainer/ppo/arctic_rl_client.py | 107 ++++++++---------- verl/workers/arctic_workers.py | 27 +---- 4 files changed, 88 insertions(+), 92 deletions(-) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh index d4bd95a7cce..594645a0363 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh @@ -12,17 +12,28 @@ export USE_ARCTIC_ZORRO=0 export CUDA_VISIBLE_DEVICES= # BSZ=1024 -BSZ=2 -MBS=2 +# MBS=256 +# UBS=32 +# ROLL_N=5 +# MAX_STEPS=100 +# PROMPT_LENGTH=512 +# RESPONSE_LENGTH=1024 + +BSZ=8 +MBS=4 UBS=2 ROLL_N=2 MAX_STEPS=4 -LR=0 -# LR=1e-6 -# LOGGER=console -LOGGER="['console','wandb']" -USE_KL_LOSS=True -# USE_KL_LOSS=False +PROMPT_LENGTH=64 +RESPONSE_LENGTH=512 + +# LR=0 +LR=1e-6 + +LOGGER=console +# LOGGER="['console','wandb']" +# USE_KL_LOSS=True +USE_KL_LOSS=False # REMOVE_PADDING=True REMOVE_PADDING=False MODEL="Qwen/Qwen3-0.6B" @@ -34,7 +45,8 @@ USE_LEGACY_WORKER_IMPL=disable NGPU_PER_NODE=1 ROLLOUT_NAME=arctic # entry point into ArcticRL USE_ARCTIC_RL=True - +# COLOCATE=True +COLOCATE=False experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_zorro${USE_ARCTIC_ZORRO}" gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) @@ -53,8 +65,8 @@ python3 -m verl.trainer.main_ppo \ data.train_files=/code/shared/gsm8k/train.parquet \ data.val_files=/code/shared/gsm8k/test.parquet \ data.train_batch_size=${BSZ} \ - data.max_prompt_length=64 \ - data.max_response_length=512 \ + data.max_prompt_length=${PROMPT_LENGTH} \ + data.max_response_length=${RESPONSE_LENGTH} \ data.filter_overlong_prompts=True \ data.truncation='error' \ data.shuffle=False \ @@ -88,6 +100,7 @@ python3 -m verl.trainer.main_ppo \ algorithm.use_kl_in_reward=False \ trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ trainer.use_arctic_rl=${USE_ARCTIC_RL} \ + arctic_rl.colocate=${COLOCATE} \ trainer.critic_warmup=0 \ trainer.logger=${LOGGER} \ trainer.experiment_name=${experiment_name} \ diff --git a/verl/trainer/config/ppo_trainer.yaml b/verl/trainer/config/ppo_trainer.yaml index 2a0779cff1c..8f9d850a658 100644 --- a/verl/trainer/config/ppo_trainer.yaml +++ b/verl/trainer/config/ppo_trainer.yaml @@ -313,3 +313,14 @@ ray_kwargs: # Path to save Ray timeline JSON for performance profiling timeline_json_file: null + + +# config for arctic rl +arctic_rl: + + # whether to use colocate mode + colocate: False + + training_gpus: 2 + sampling_gpus: 1 + log_prob_gpus: 1 diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index adaf1ddcef4..ec68fd7f9c1 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -180,9 +180,10 @@ def is_zorro_enabled(self): def initialize(self, model_name: str): from arctic_training.arctic_rl import ArcticRLClient, ArcticRLClientConfig - n_gpus = self.config.trainer.n_gpus_per_node - #n_gpus = 2 - colocate = self.config.actor_rollout_ref.hybrid_engine + n_training_gpus = self.config.arctic_rl.get("training_gpus", self.config.trainer.n_gpus_per_node) + n_sampling_gpus = self.config.arctic_rl.get("sampling_gpus", self.config.trainer.n_gpus_per_node) + n_log_prob_gpus = self.config.arctic_rl.get("log_prob_gpus", self.config.trainer.n_gpus_per_node) + colocate = self.config.arctic_rl.get("colocate", False) attn_implementation = self.config.actor_rollout_ref.model.override_config.get( 'attn_implementation', 'eager' ) @@ -193,7 +194,7 @@ def initialize(self, model_name: str): micro_batch_size = actor_cfg.ppo_micro_batch_size_per_gpu or 1 train_batch_size = data_cfg.train_batch_size - grad_accum_steps = max(1, train_batch_size // (micro_batch_size * n_gpus)) + grad_accum_steps = max(1, train_batch_size // (micro_batch_size * n_training_gpus)) seq_parallel_size = actor_cfg.fsdp_config.get("ulysses_sequence_parallel_size", 1) max_length = data_cfg.max_prompt_length + data_cfg.max_response_length @@ -213,9 +214,9 @@ def initialize(self, model_name: str): host="localhost", port=7000, backend="local", - training_gpus=n_gpus, - sample_gpus=n_gpus, - log_prob_gpus=n_gpus, + training_gpus=n_training_gpus, + sampling_gpus=n_sampling_gpus, + log_prob_gpus=n_log_prob_gpus, colocate=colocate, log_prob_engine="deepspeed", model_name=model_name, @@ -244,9 +245,9 @@ def initialize(self, model_name: str): # ArcticRLClient is constructed as a ray remote actor with num_gpus=0, # which causes CUDA_VISIBLE_DEVICES to be empty. if colocate: - num_visible = n_gpus + num_visible = n_training_gpus + n_sampling_gpus + n_log_prob_gpus else: - num_visible = rl_config.training_gpus + rl_config.sample_gpus + rl_config.log_prob_gpus + num_visible = rl_config.training_gpus + rl_config.sampling_gpus + rl_config.log_prob_gpus os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in range(num_visible)) self._client = ArcticRLClient(rl_config) @@ -265,45 +266,28 @@ def generate(self, prompt_ids, sampling_params) -> list: merged_params = {**self._default_sampling_params, **sampling_params} return self._client.generate(prompts=prompts, sampling_params=merged_params) - def compute_ref_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict = None): - return self.compute_log_prob(dss_batch_dict, post_process_inputs) - - def compute_log_prob(self, dss_batch_dict: dict, post_process_inputs: dict = None): - print(f"[ArcticRLWrapper] compute_log_prob INPUT: " - f"{{{', '.join(f'{k}: {v.shape}' for k, v in dss_batch_dict.items() if isinstance(v, torch.Tensor))}}}") - batch = { - "kwargs": dss_batch_dict, - "context": {"input_ids": dss_batch_dict["input_ids"]}, - "processing": { - "post": ["compute_logprobs"], - "loss_fn": None, - }, - } - result = self._client.fwd_no_grad(batch) - outputs = result.get("model_outputs", result) - - log_probs = outputs.get("logprobs") - if log_probs is not None and not isinstance(log_probs, torch.Tensor): - log_probs = torch.tensor(log_probs) - - # The pipeline doesn't return true entropy; approximate as - # -logprobs to match what grpo_loss uses internally (grpo.py:249). - entropy = -log_probs if log_probs is not None else None - print(f"[ArcticRLWrapper] compute_log_prob OUTPUT: " - f"entropy={entropy.shape if entropy is not None else None} " - f"log_probs={log_probs.shape if log_probs is not None else None}") - return entropy, log_probs + def compute_ref_log_prob(self, payload: dict): + payload["processing"] = {"post": ["compute_logprobs", "compute_entropy"], "loss_fn": None} + response = self._client.fwd_no_grad(payload, reference_model=True) + response["batch"]["log_probs"] = response["batch"].pop("logprobs") + print(f"[ArcticRLWrapper] compute_ref_log_prob OUTPUT: {response.keys()=}") + return response - def update_actor(self, dss_batch_dict: dict, post_process_inputs: dict): - extra = post_process_inputs.get("extra_inputs", {}) - seq_len = dss_batch_dict["input_ids"].shape[-1] - print(f"[ArcticRLWrapper] update_actor INPUT: " - f"{{{', '.join(f'{k}: {v.shape}' for k, v in dss_batch_dict.items() if isinstance(v, torch.Tensor))}}} " - f"| extra: {{{', '.join(f'{k}: {v.shape}' for k, v in extra.items() if isinstance(v, torch.Tensor))}}}") + def compute_log_prob(self, payload: dict): + payload["processing"] = {"post": ["compute_logprobs", "compute_entropy"], "loss_fn": None} + response = self._client.fwd_no_grad(payload, reference_model=False) + response["batch"]["log_probs"] = response["batch"].pop("logprobs") + print(f"[ArcticRLWrapper] compute_log_prob OUTPUT: {response.keys()=}") + return response - def _left_pad(t: torch.Tensor) -> torch.Tensor: + def update_actor(self, payload: dict): + payload["processing"] = { + "post": ["apply_temperature", "compute_logprobs", "compute_entropy"], + "loss_fn": "verl_grpo" + } + def _left_pad(t: torch.Tensor, seq_len: int) -> torch.Tensor: """Left-pad a response-only tensor to full sequence length with zeros.""" pad_len = seq_len - t.shape[-1] if pad_len <= 0: @@ -311,25 +295,28 @@ def _left_pad(t: torch.Tensor) -> torch.Tensor: pad = torch.zeros(*t.shape[:-1], pad_len, dtype=t.dtype, device=t.device) return torch.cat([pad, t], dim=-1) - context = { - "input_ids": dss_batch_dict["input_ids"], - "old_log_probs_shifted": _left_pad(extra["old_log_probs"]), - "advantages": _left_pad(extra["advantages"]), - "loss_mask": _left_pad(extra["response_mask"]), - } - print(f"[ArcticRLWrapper] update_actor CONTEXT: " - f"{{{', '.join(f'{k}: {v.shape}' for k, v in context.items() if isinstance(v, torch.Tensor))}}}") - batch = {"kwargs": dss_batch_dict, "context": context} + seq_len = payload["batch"]["input_ids"].shape[-1] + for name in ["old_log_probs", "advantages", "response_mask", "ref_log_prob"]: + if name in payload["batch"]: + payload["batch"][name] = _left_pad(payload["batch"][name], seq_len) - result = self._client.fwd_bwd(batch, processing={"loss_fn": "grpo", "post": ["compute_logprobs"]}) - self._client.step() + fwd_bwd_response = self._client.fwd_bwd(payload) + print(f"[ArcticRLWrapper] update_actor OUTPUT: {fwd_bwd_response.keys()=}") + step_response = self._client.step() + print(f"[ArcticRLWrapper] update_actor STEP OUTPUT: {step_response.keys()=}") + step_response["metrics"].update(**fwd_bwd_response["metrics"]) + return step_response - loss = result.get("avg_loss", 0.0) - raw_metrics = result.get("post_process_outputs", {}) - metrics = {k: v if isinstance(v, list) else [v] for k, v in raw_metrics.items()} - print(f"[ArcticRLWrapper] update_actor OUTPUT: loss={loss} metrics={metrics}") - return loss, metrics + def save_checkpoint(self): + response = self._client.save_checkpoint() + print(f"[ArcticRLClientWrapper] save_checkpoint OUTPUT: {response.keys()=}") + return response + + def update_weights(self): + response = self._client.sync_weights() + print(f"[ArcticRLClientWrapper] update_weights OUTPUT: {response.keys()=}") + return response def destroy(self): if self._client is not None: diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 052efa3e33d..0d032eb18dd 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -166,23 +166,6 @@ def prepare_model_inputs_remove_padding(micro_batch: TensorDict): return model_inputs, output_args -def prepare_extra_inputs(data: TensorDict) -> dict: - extra_inputs = dict( - prompts=data["prompts"], - responses=data["responses"], - attention_mask=data["attention_mask"], - max_token_len_per_gpu=data["max_token_len_per_gpu"], - global_batch_size=data["global_batch_size"], - response_mask=data["response_mask"], - old_log_probs=data["old_log_probs"], - advantages=data["advantages"], - ref_log_prob=data["ref_log_prob"], - rollout_is_weights=data.get("rollout_is_weights", None), - batch_num_tokens=data["loss_mask"].sum(), - ) - - return extra_inputs - def prepand_max_prompt_len_zeros(tensor: Tensor, max_prompt_len): prepand = torch.zeros([tensor.shape[0], max_prompt_len], dtype=torch.int64, device=tensor.device) return torch.cat([prepand, tensor], dim=1) @@ -430,7 +413,7 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: # from verl.utils.tensordict_utils import chunk_tensordict # batch = chunk_tensordict(data, 1) - print(f"update_actor data: {data}") + # print(f"update_actor data: {data}") # XXX: fix me input_ids = data['input_ids'] @@ -478,7 +461,7 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: if self.actor_config.use_kl_loss: batch["ref_log_prob"] = data["ref_log_prob"] - print(f"{batch=}") + # print(f"{batch=}") # TODO: move to init since globally constant meta = dict( @@ -520,7 +503,7 @@ def safe_serialize(obj): from verl.utils.metric import AggregationType, Metric # XXX: fix me - we need to aggregate the metrics - metrics = {k:Metric(value=v[0], aggregation=AggregationType.MEAN) for k,v in metrics.items()} + metrics = {k:Metric(value=v[0] if isinstance(v, list) else v, aggregation=AggregationType.MEAN) for k,v in metrics.items()} metrics["lr"] = metrics.pop("last_lr") delta_time = timer.last @@ -697,7 +680,7 @@ def _update_config_params(self, data: TensorDict): def compute_any_log_prob(self, data: TensorDict, compute_log_prob_fn) -> TensorDict: - print(f"compute_ref_log_prob data: {data}") + # print(f"compute_ref_log_prob data: {data}") batch, max_prompt_len, max_response_len = prepare_padded_dss_batch_dict(data, self.pad_token_id) self._update_config_params(data) @@ -768,6 +751,7 @@ def load_checkpoint(self, local_path, hdfs_path=None, del_local_after_load=False @register(dispatch_mode=Dispatch.ONE_TO_ALL) def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): assert "actor" in self.role, "save_checkpoint only support actor role" + ray.get(self.arctic_rl_client.save_checkpoint.remote()) return # TODO: Update Weights API @@ -780,6 +764,7 @@ async def update_weights(self, global_steps: int = None): - after update_weights: rollout should be in wake_up mode. 2. For async training with disaggregated trainer and rollout, send_weights only by checkpoint engine. """ + ray.get(self.arctic_rl_client.update_weights.remote()) return # TODO: CheckpointManager API Begin From fd1e63df543fbcd73a4dac5fa34432290c7e8d34 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Wed, 15 Apr 2026 16:12:08 +0000 Subject: [PATCH 36/58] Fix multi-gpu mismatch for train/ & log_prob --- .../run_arctic_gsm8k_grpo_zorro_no.sh | 9 +++-- examples/arctic_rl/run_gsm8k_grpo.sh | 39 ++++++++++++++----- verl/trainer/config/ppo_trainer.yaml | 2 +- verl/trainer/ppo/arctic_rl_client.py | 35 +++++++++++------ verl/workers/arctic_workers.py | 4 +- verl/workers/engine/utils.py | 7 +++- verl/workers/utils/padding.py | 16 ++++---- 7 files changed, 75 insertions(+), 37 deletions(-) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh index 594645a0363..b57687242d4 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh @@ -20,12 +20,12 @@ export CUDA_VISIBLE_DEVICES= # RESPONSE_LENGTH=1024 BSZ=8 -MBS=4 +MBS=2 UBS=2 ROLL_N=2 MAX_STEPS=4 -PROMPT_LENGTH=64 -RESPONSE_LENGTH=512 +PROMPT_LENGTH=512 +RESPONSE_LENGTH=1024 # LR=0 LR=1e-6 @@ -101,6 +101,9 @@ python3 -m verl.trainer.main_ppo \ trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ trainer.use_arctic_rl=${USE_ARCTIC_RL} \ arctic_rl.colocate=${COLOCATE} \ + arctic_rl.training_gpus=1\ + arctic_rl.sampling_gpus=2\ + arctic_rl.log_prob_gpus=1\ trainer.critic_warmup=0 \ trainer.logger=${LOGGER} \ trainer.experiment_name=${experiment_name} \ diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh index 2328483f738..c9f397bc1bc 100755 --- a/examples/arctic_rl/run_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_gsm8k_grpo.sh @@ -6,17 +6,27 @@ export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 # BSZ=1024 -BSZ=2 +# MBS=256 +# UBS=32 +ROLL_N=5 +# MAX_STEPS=100 +PROMPT_LENGTH=512 +RESPONSE_LENGTH=1024 +BSZ=8 MBS=2 UBS=2 -ROLL_N=2 +# ROLL_N=2 MAX_STEPS=4 +# PROMPT_LENGTH=64 +# RESPONSE_LENGTH=512 + # LR=0 LR=1e-6 -LOGGER=console -# LOGGER="['console','wandb']" -USE_KL_LOSS=True -# USE_KL_LOSS=False + +# LOGGER=console +LOGGER="['console','wandb']" +# USE_KL_LOSS=True +USE_KL_LOSS=False # REMOVE_PADDING=True REMOVE_PADDING=False MODEL="Qwen/Qwen3-0.6B" @@ -25,18 +35,28 @@ STRATEGY="fsdp2" PYTHONUNBUFFERED=1 HYDRA_FULL_ERROR=1 USE_LEGACY_WORKER_IMPL=disable -NGPU_PER_NODE=1 +NGPU_PER_NODE=4 ROLLOUT_NAME=vllm experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_baseline" +gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) +if [[ $gpu_name == *"H200"* ]]; then + echo "Running on Hopper" + flash_attention_v=flash_attention_3 +elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then + echo "Running on Blackwell" + flash_attention_v=flash_attention_2 +else + echo "Running on unknown: $gpu_name; don't know which FA version to use" +fi python3 -m verl.trainer.main_ppo \ algorithm.adv_estimator=grpo \ data.train_files=/code/shared/gsm8k/train.parquet \ data.val_files=/code/shared/gsm8k/test.parquet \ data.train_batch_size=${BSZ} \ - data.max_prompt_length=64 \ - data.max_response_length=512 \ + data.max_prompt_length=${PROMPT_LENGTH} \ + data.max_response_length=${RESPONSE_LENGTH} \ data.filter_overlong_prompts=True \ data.truncation='error' \ data.shuffle=False \ @@ -54,6 +74,7 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.kl_loss_type=low_var_kl \ actor_rollout_ref.actor.entropy_coeff=0 \ actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ actor_rollout_ref.actor.strategy=${STRATEGY} \ actor_rollout_ref.actor.fsdp_config.param_offload=False \ actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ diff --git a/verl/trainer/config/ppo_trainer.yaml b/verl/trainer/config/ppo_trainer.yaml index 8f9d850a658..612a07eae9e 100644 --- a/verl/trainer/config/ppo_trainer.yaml +++ b/verl/trainer/config/ppo_trainer.yaml @@ -321,6 +321,6 @@ arctic_rl: # whether to use colocate mode colocate: False - training_gpus: 2 + training_gpus: 1 sampling_gpus: 1 log_prob_gpus: 1 diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index ec68fd7f9c1..0b8be109f11 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -1,4 +1,5 @@ import os +from typing import Any import torch from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer from deepspeed.utils import OnDevice @@ -177,6 +178,22 @@ def __init__(self, config): def is_zorro_enabled(self): return self.use_zorro + def _create_ds_config(self, n_gpus: int) -> dict[str, Any]: + actor_cfg = self.config.actor_rollout_ref.actor + data_cfg = self.config.data + + micro_batch_size = actor_cfg.ppo_micro_batch_size_per_gpu or 1 + train_batch_size = data_cfg.train_batch_size + grad_accum_steps = max(1, train_batch_size // (micro_batch_size * n_gpus)) + train_seq_parallel_size = actor_cfg.fsdp_config.get("ulysses_sequence_parallel_size", 1) + return { + "train_micro_batch_size_per_gpu": micro_batch_size, + "train_batch_size": train_batch_size, + "gradient_accumulation_steps": grad_accum_steps, + "sequence_parallel_size": train_seq_parallel_size, + "zero_optimization": {"stage": 1}, + } + def initialize(self, model_name: str): from arctic_training.arctic_rl import ArcticRLClient, ArcticRLClientConfig @@ -192,10 +209,6 @@ def initialize(self, model_name: str): optim_cfg = actor_cfg.optim data_cfg = self.config.data - micro_batch_size = actor_cfg.ppo_micro_batch_size_per_gpu or 1 - train_batch_size = data_cfg.train_batch_size - grad_accum_steps = max(1, train_batch_size // (micro_batch_size * n_training_gpus)) - seq_parallel_size = actor_cfg.fsdp_config.get("ulysses_sequence_parallel_size", 1) max_length = data_cfg.max_prompt_length + data_cfg.max_response_length rollout_cfg = self.config.actor_rollout_ref.rollout @@ -220,13 +233,8 @@ def initialize(self, model_name: str): colocate=colocate, log_prob_engine="deepspeed", model_name=model_name, - ds_config={ - "train_micro_batch_size_per_gpu": micro_batch_size, - "train_batch_size": train_batch_size, - "gradient_accumulation_steps": grad_accum_steps, - "sequence_parallel_size": seq_parallel_size, - "zero_optimization": {"stage": 1}, - }, + ds_config=self._create_ds_config(n_training_gpus), + log_prob_ds_config=self._create_ds_config(n_log_prob_gpus), training_config={ "optimizer": { "lr": optim_cfg.lr, @@ -285,7 +293,8 @@ def compute_log_prob(self, payload: dict): def update_actor(self, payload: dict): payload["processing"] = { "post": ["apply_temperature", "compute_logprobs", "compute_entropy"], - "loss_fn": "verl_grpo" + #"loss_fn": "verl_grpo" + "loss_fn": "grpo" } def _left_pad(t: torch.Tensor, seq_len: int) -> torch.Tensor: """Left-pad a response-only tensor to full sequence length with zeros.""" @@ -300,6 +309,8 @@ def _left_pad(t: torch.Tensor, seq_len: int) -> torch.Tensor: if name in payload["batch"]: payload["batch"][name] = _left_pad(payload["batch"][name], seq_len) + payload["batch"]["loss_mask"] = payload["batch"]["response_mask"] + fwd_bwd_response = self._client.fwd_bwd(payload) print(f"[ArcticRLWrapper] update_actor OUTPUT: {fwd_bwd_response.keys()=}") step_response = self._client.step() diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 0d032eb18dd..88032b3b16e 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -479,8 +479,8 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: # we need to serialize the config object to dict # dataclasses.asdict only returns keys that are defined at init (vars will do more) - but perhaps we want `asdict`? actor_config_as_dict = vars(self.actor_config) - print(f"update_actor: {self.actor_config=}") - print(f"update_actor: {actor_config_as_dict=}") + # print(f"update_actor: {self.actor_config=}") + # print(f"update_actor: {actor_config_as_dict=}") import json def safe_serialize(obj): return json.loads(json.dumps(obj, default=lambda o: None)) diff --git a/verl/workers/engine/utils.py b/verl/workers/engine/utils.py index ebc5d430d81..2547c747e23 100644 --- a/verl/workers/engine/utils.py +++ b/verl/workers/engine/utils.py @@ -89,9 +89,12 @@ def prepare_micro_batches( else: total_data_size = len(data) micro_batch_size_per_gpu = data["micro_batch_size_per_gpu"] + # assert total_data_size % (force_group_size * micro_batch_size_per_gpu) == 0, ( + # "data size must be divisible by force_group_size * micro_batch_size_per_gpu" + # ) assert total_data_size % (force_group_size * micro_batch_size_per_gpu) == 0, ( - "data size must be divisible by force_group_size * micro_batch_size_per_gpu" - ) + f"data size {total_data_size} must be divisible by force_group_size {force_group_size} * micro_batch_size_per_gpu {micro_batch_size_per_gpu}" + ) micro_batches = tu.chunk_tensordict(data, total_data_size // (micro_batch_size_per_gpu * force_group_size)) batch_idx_list = None return micro_batches, batch_idx_list diff --git a/verl/workers/utils/padding.py b/verl/workers/utils/padding.py index 69ed8de90f5..30ce9312aa8 100644 --- a/verl/workers/utils/padding.py +++ b/verl/workers/utils/padding.py @@ -120,14 +120,14 @@ def no_padding_2_padding(tensor: torch.Tensor, data: TensorDict) -> torch.Tensor sequence_lens = prompt_lens + response_lens sequence_offsets = sequence_lens.cumsum(dim=0) - print(f"{data=}") - print(f"{prompt_lens=}") - print(f"{response_lens=}") - print(f"{response_lens=}") - print(f"{max_response_len=}") - print(f"{sequence_offsets=}") - print(f"{values=}") - print(f"{values.shape=}") + # print(f"{data=}") + # print(f"{prompt_lens=}") + # print(f"{response_lens=}") + # print(f"{response_lens=}") + # print(f"{max_response_len=}") + # print(f"{sequence_offsets=}") + # print(f"{values=}") + # print(f"{values.shape=}") assert sequence_offsets[-1].item() == values.shape[0], f"{sequence_offsets[-1].item()} != {values.shape[0]}" response_list = [] From f4fc13f43a72aa2a2e008f253b68dded7c590076 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Wed, 15 Apr 2026 18:09:56 +0000 Subject: [PATCH 37/58] Add setup --- .../arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh | 6 +++--- examples/arctic_rl/run_gsm8k_grpo.sh | 2 +- scripts/arctic_rl/install.sh | 5 +++++ scripts/arctic_rl/setup_repos.sh | 4 ++++ verl/workers/utils/padding.py | 16 ++++++++-------- 5 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 scripts/arctic_rl/install.sh create mode 100644 scripts/arctic_rl/setup_repos.sh diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh index b57687242d4..2d221c382c1 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh @@ -19,10 +19,10 @@ export CUDA_VISIBLE_DEVICES= # PROMPT_LENGTH=512 # RESPONSE_LENGTH=1024 -BSZ=8 -MBS=2 +BSZ=4 +MBS=4 UBS=2 -ROLL_N=2 +ROLL_N=5 MAX_STEPS=4 PROMPT_LENGTH=512 RESPONSE_LENGTH=1024 diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh index c9f397bc1bc..3abfc33c7ae 100755 --- a/examples/arctic_rl/run_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_gsm8k_grpo.sh @@ -13,7 +13,7 @@ ROLL_N=5 PROMPT_LENGTH=512 RESPONSE_LENGTH=1024 BSZ=8 -MBS=2 +MBS=8 UBS=2 # ROLL_N=2 MAX_STEPS=4 diff --git a/scripts/arctic_rl/install.sh b/scripts/arctic_rl/install.sh new file mode 100644 index 00000000000..1294c76819a --- /dev/null +++ b/scripts/arctic_rl/install.sh @@ -0,0 +1,5 @@ +uv pip install -e "./ArcticInference-internal[server]" +uv pip install -e ./dss-client +uv pip install -e ./ArcticTraining-dss +cd arctic-verl +/code/shared/verl_snowrlhf/install-h200.sh diff --git a/scripts/arctic_rl/setup_repos.sh b/scripts/arctic_rl/setup_repos.sh new file mode 100644 index 00000000000..9d2c905bfd0 --- /dev/null +++ b/scripts/arctic_rl/setup_repos.sh @@ -0,0 +1,4 @@ +git clone -b verl_integration https://github.com/snowflake-eng/dss-client.git +git clone -b tunji/verl_integration https://github.com/snowflake-eng/ArcticTraining-dss.git +git clone -b public https://github.com/snowflake-eng/ArcticInference-internal.git +git clone -b tunji/arl_client https://github.com/snowflake-eng/arctic-verl.git diff --git a/verl/workers/utils/padding.py b/verl/workers/utils/padding.py index 30ce9312aa8..cafa0d310a1 100644 --- a/verl/workers/utils/padding.py +++ b/verl/workers/utils/padding.py @@ -191,14 +191,14 @@ def no_padding_2_padding_prompt_response(tensor: torch.Tensor, data: TensorDict, sequence_lens = prompt_lens + response_lens sequence_offsets = sequence_lens.cumsum(dim=0) - print(f"{data=}") - print(f"{prompt_lens=}") - print(f"{response_lens=}") - print(f"{max_prompt_len=}") - print(f"{max_response_len=}") - print(f"{sequence_offsets=}") - print(f"{values=}") - print(f"{values.shape=}") + # print(f"{data=}") + # print(f"{prompt_lens=}") + # print(f"{response_lens=}") + # print(f"{max_prompt_len=}") + # print(f"{max_response_len=}") + # print(f"{sequence_offsets=}") + # print(f"{values=}") + # print(f"{values.shape=}") assert sequence_offsets[-1].item() == values.shape[0], f"{sequence_offsets[-1].item()} != {values.shape[0]}" input_ids_list = [] From 370e6aabe7a05df26d47f1d82a718c59152aafa5 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 16 Apr 2026 17:26:02 +0000 Subject: [PATCH 38/58] integrate zorro Signed-off-by: Stas Bekman --- .../run_arctic_gsm8k_grpo_zorro_no.sh | 70 +++++++------- .../run_arctic_gsm8k_grpo_zorro_yes.sh | 93 ++++++++++++------- verl/trainer/config/ppo_trainer.yaml | 4 +- verl/trainer/ppo/arctic_rl_client.py | 41 +++++++- 4 files changed, 136 insertions(+), 72 deletions(-) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh index 2d221c382c1..fb2f6d30e88 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh @@ -19,13 +19,21 @@ export CUDA_VISIBLE_DEVICES= # PROMPT_LENGTH=512 # RESPONSE_LENGTH=1024 -BSZ=4 -MBS=4 +# BSZ=4 +# MBS=4 +# UBS=2 +# ROLL_N=5 +# MAX_STEPS=4 +# PROMPT_LENGTH=512 +# RESPONSE_LENGTH=1024 + +BSZ=2 +MBS=2 UBS=2 -ROLL_N=5 +ROLL_N=4 MAX_STEPS=4 -PROMPT_LENGTH=512 -RESPONSE_LENGTH=1024 +PROMPT_LENGTH=64 +RESPONSE_LENGTH=512 # LR=0 LR=1e-6 @@ -39,15 +47,13 @@ REMOVE_PADDING=False MODEL="Qwen/Qwen3-0.6B" # STRATEGY="fsdp" STRATEGY="fsdp2" -PYTHONUNBUFFERED=1 -HYDRA_FULL_ERROR=1 USE_LEGACY_WORKER_IMPL=disable NGPU_PER_NODE=1 ROLLOUT_NAME=arctic # entry point into ArcticRL USE_ARCTIC_RL=True # COLOCATE=True COLOCATE=False -experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_zorro${USE_ARCTIC_ZORRO}" +experiment_name="qwen3-0.6B_ngpu$NGPU_PER_NODE_gbs$BSZ_rolln$ROLL_N_zorro$USE_ARCTIC_ZORRO" gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) if [[ $gpu_name == *"H200"* ]]; then @@ -64,9 +70,9 @@ python3 -m verl.trainer.main_ppo \ algorithm.adv_estimator=grpo \ data.train_files=/code/shared/gsm8k/train.parquet \ data.val_files=/code/shared/gsm8k/test.parquet \ - data.train_batch_size=${BSZ} \ - data.max_prompt_length=${PROMPT_LENGTH} \ - data.max_response_length=${RESPONSE_LENGTH} \ + data.train_batch_size=$BSZ \ + data.max_prompt_length=$PROMPT_LENGTH \ + data.max_response_length=$RESPONSE_LENGTH \ data.filter_overlong_prompts=True \ data.truncation='error' \ data.shuffle=False \ @@ -74,46 +80,46 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.data_loader_seed=42 \ reward.num_workers=1 \ actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.model.path=${MODEL} \ - actor_rollout_ref.actor.optim.lr=${LR} \ - actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ - actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ + actor_rollout_ref.model.path=$MODEL \ + actor_rollout_ref.actor.optim.lr=$LR \ + actor_rollout_ref.model.use_remove_padding=$REMOVE_PADDING \ + actor_rollout_ref.actor.ppo_mini_batch_size=$MBS \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=$UBS \ + actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ actor_rollout_ref.actor.kl_loss_coef=0.001 \ actor_rollout_ref.actor.kl_loss_type=low_var_kl \ actor_rollout_ref.actor.entropy_coeff=0 \ actor_rollout_ref.model.enable_gradient_checkpointing=True \ +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.actor.strategy=${STRATEGY} \ + actor_rollout_ref.actor.strategy=$STRATEGY \ actor_rollout_ref.actor.fsdp_config.param_offload=False \ actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=$UBS \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ actor_rollout_ref.rollout.enforce_eager=True \ - actor_rollout_ref.rollout.n=${ROLL_N} \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.n=$ROLL_N \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=$UBS \ actor_rollout_ref.ref.fsdp_config.param_offload=False \ - actor_rollout_ref.ref.strategy=${STRATEGY} \ + actor_rollout_ref.ref.strategy=$STRATEGY \ algorithm.use_kl_in_reward=False \ - trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ - trainer.use_arctic_rl=${USE_ARCTIC_RL} \ - arctic_rl.colocate=${COLOCATE} \ + trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ + trainer.use_arctic_rl=$USE_ARCTIC_RL \ + arctic_rl.colocate=$COLOCATE \ arctic_rl.training_gpus=1\ arctic_rl.sampling_gpus=2\ arctic_rl.log_prob_gpus=1\ trainer.critic_warmup=0 \ - trainer.logger=${LOGGER} \ - trainer.experiment_name=${experiment_name} \ + trainer.logger=$LOGGER \ + trainer.experiment_name=$experiment_name \ trainer.project_name='verl_arctic_grpo_gsm8k' \ trainer.val_before_train=False \ - trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.n_gpus_per_node=$NGPU_PER_NODE \ trainer.nnodes=1 \ trainer.save_freq=-1 \ trainer.test_freq=-1 \ - trainer.total_training_steps=${MAX_STEPS} \ - trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log + trainer.total_training_steps=$MAX_STEPS \ + trainer.total_epochs=15 $@ 2>&1 | tee $experiment_name.log - # trainer.total_training_steps=${MAX_STEPS} \ + # trainer.total_training_steps=$MAX_STEPS \ diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh index 794d8adfe5f..b8e04007738 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh @@ -12,30 +12,48 @@ export USE_ARCTIC_ZORRO=1 export CUDA_VISIBLE_DEVICES= # BSZ=1024 -BSZ=2 -MBS=2 +BSZ=4 +MBS=4 UBS=2 ROLL_N=2 MAX_STEPS=4 -LR=0 -# LR=1e-6 -# LOGGER=console -LOGGER="['console','wandb']" -USE_KL_LOSS=True -# USE_KL_LOSS=False +PROMPT_LENGTH=64 +RESPONSE_LENGTH=512 + +# BSZ=1 +# MBS=1 +# UBS=1 +# ROLL_N=16 +# MAX_STEPS=4 +# PROMPT_LENGTH=1024 +# RESPONSE_LENGTH=2048 + +# BSZ=2 +# MBS=2 +# UBS=2 +# ROLL_N=4 +# MAX_STEPS=4 +# PROMPT_LENGTH=64 +# RESPONSE_LENGTH=512 + +# LR=0 +LR=1e-6 + +LOGGER=console +# LOGGER="['console','wandb']" +# USE_KL_LOSS=True +USE_KL_LOSS=False # REMOVE_PADDING=True REMOVE_PADDING=False MODEL="Qwen/Qwen3-0.6B" # STRATEGY="fsdp" STRATEGY="fsdp2" -PYTHONUNBUFFERED=1 -HYDRA_FULL_ERROR=1 USE_LEGACY_WORKER_IMPL=disable NGPU_PER_NODE=1 ROLLOUT_NAME=arctic # entry point into ArcticRL USE_ARCTIC_RL=True - -experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_zorro${USE_ARCTIC_ZORRO}" +COLOCATE=False +experiment_name="qwen3-0.6B_ngpu$NGPU_PER_NODE_gbs$BSZ_rolln$ROLL_N_zorro$USE_ARCTIC_ZORRO" gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) if [[ $gpu_name == *"H200"* ]]; then @@ -52,9 +70,9 @@ python3 -m verl.trainer.main_ppo \ algorithm.adv_estimator=grpo \ data.train_files=/code/shared/gsm8k/train.parquet \ data.val_files=/code/shared/gsm8k/test.parquet \ - data.train_batch_size=${BSZ} \ - data.max_prompt_length=64 \ - data.max_response_length=512 \ + data.train_batch_size=$BSZ \ + data.max_prompt_length=$PROMPT_LENGTH \ + data.max_response_length=$RESPONSE_LENGTH \ data.filter_overlong_prompts=True \ data.truncation='error' \ data.shuffle=False \ @@ -62,42 +80,47 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.data_loader_seed=42 \ reward.num_workers=1 \ actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.model.path=${MODEL} \ - actor_rollout_ref.actor.optim.lr=${LR} \ - actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ - actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ + actor_rollout_ref.model.path=$MODEL \ + actor_rollout_ref.actor.optim.lr=$LR \ + actor_rollout_ref.model.use_remove_padding=$REMOVE_PADDING \ + actor_rollout_ref.actor.ppo_mini_batch_size=$MBS \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=$UBS \ + actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ actor_rollout_ref.actor.kl_loss_coef=0.001 \ actor_rollout_ref.actor.kl_loss_type=low_var_kl \ actor_rollout_ref.actor.entropy_coeff=0 \ actor_rollout_ref.model.enable_gradient_checkpointing=True \ +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.actor.strategy=${STRATEGY} \ + actor_rollout_ref.actor.strategy=$STRATEGY \ actor_rollout_ref.actor.fsdp_config.param_offload=False \ actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=$UBS \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ actor_rollout_ref.rollout.enforce_eager=True \ - actor_rollout_ref.rollout.n=${ROLL_N} \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.n=$ROLL_N \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=$UBS \ actor_rollout_ref.ref.fsdp_config.param_offload=False \ - actor_rollout_ref.ref.strategy=${STRATEGY} \ + actor_rollout_ref.ref.strategy=$STRATEGY \ algorithm.use_kl_in_reward=False \ - trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ - trainer.use_arctic_rl=${USE_ARCTIC_RL} \ + trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ + trainer.use_arctic_rl=$USE_ARCTIC_RL \ + arctic_rl.colocate=$COLOCATE \ + arctic_rl.training_gpus=1\ + arctic_rl.sampling_gpus=2\ + arctic_rl.log_prob_gpus=1\ + arctic_rl.use_zorro=True \ trainer.critic_warmup=0 \ - trainer.logger=${LOGGER} \ - trainer.experiment_name=${experiment_name} \ + trainer.logger=$LOGGER \ + trainer.experiment_name=$experiment_name \ trainer.project_name='verl_arctic_grpo_gsm8k' \ trainer.val_before_train=False \ - trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.n_gpus_per_node=$NGPU_PER_NODE \ trainer.nnodes=1 \ trainer.save_freq=-1 \ trainer.test_freq=-1 \ - trainer.total_training_steps=${MAX_STEPS} \ - trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log + trainer.total_training_steps=$MAX_STEPS \ + trainer.total_epochs=15 $@ 2>&1 | tee $experiment_name.log - # trainer.total_training_steps=${MAX_STEPS} \ + # trainer.total_training_steps=$MAX_STEPS \ diff --git a/verl/trainer/config/ppo_trainer.yaml b/verl/trainer/config/ppo_trainer.yaml index 612a07eae9e..e2908368851 100644 --- a/verl/trainer/config/ppo_trainer.yaml +++ b/verl/trainer/config/ppo_trainer.yaml @@ -203,7 +203,7 @@ trainer: # mode: "auto", "enable", or "disable" use_legacy_worker_impl: auto - # whether to use arctic rl + # whether to use arctic rl use_arctic_rl: False # profiler configs @@ -324,3 +324,5 @@ arctic_rl: training_gpus: 1 sampling_gpus: 1 log_prob_gpus: 1 + + use_zorro: False diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 0b8be109f11..2ac27862558 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -38,7 +38,7 @@ def __init__(self, config): config: verl's full config """ self.config = config - self.use_zorro = USE_ARCTIC_ZORRO + self.use_zorro = self.config.arctic_rl.use_zorro #print(f"ArcticRLClient4VeRL {config=}") self.arctic_inference_client = DSSInferenceClient(dss_server_url="http://localhost:7000") @@ -97,6 +97,20 @@ def initialize(self, model_name: str): "attn_implementation": attn_implementation, } + if self.is_zorro_enabled(): + # XXX: can't find where it's configured + use_unpad = True + + training_config.update( + use_zorro=True, + response_len=self.config.data.max_response_length, + max_token_len=self.config.actor_rollout_ref.rollout.max_num_batched_tokens, + rollout_n=self.config.actor_rollout_ref.rollout.n, + temperature=self.config.actor_rollout_ref.rollout.temperature, + use_unpad=use_unpad, + ) + #print(f"{training_config=}") + self.training_engine = self.arctic_training_client.initialize( model=create_meta_model(model_name), ds_config=ds_config, @@ -121,7 +135,7 @@ def compute_ref_log_prob(self, payload: dict): # log_probs = torch.tensor(log_probs).squeeze() print(f"arctic_rl_client.compute_ref_log_prob: {response['batch']['entropy'].shape=}, {response['batch']['log_probs'].shape=}") return response - + def compute_log_prob(self, payload: dict): # XXX: somehow we need to differentiate which model is this called on ref vs actor - at the moment it's always actor hardcoded @@ -173,7 +187,7 @@ def __init__(self, config): self.config = config self._client = None self.tokenizer = None - self.use_zorro = USE_ARCTIC_ZORRO + self.use_zorro = self.config.arctic_rl.use_zorro def is_zorro_enabled(self): return self.use_zorro @@ -194,6 +208,24 @@ def _create_ds_config(self, n_gpus: int) -> dict[str, Any]: "zero_optimization": {"stage": 1}, } + def _create_ds_worker_config(self): + + if self.is_zorro_enabled(): + # XXX: can't find where it's configured + use_unpad = True + + return dict( + use_zorro=True, + response_len=self.config.data.max_response_length, + max_token_len=self.config.actor_rollout_ref.rollout.max_num_batched_tokens, + rollout_n=self.config.actor_rollout_ref.rollout.n, + temperature=self.config.actor_rollout_ref.rollout.temperature, + use_unpad=use_unpad, + ) + else: + return {} + + def initialize(self, model_name: str): from arctic_training.arctic_rl import ArcticRLClient, ArcticRLClientConfig @@ -247,6 +279,7 @@ def initialize(self, model_name: str): "model_config": None, "attn_implementation": attn_implementation, }, + ds_worker_config=self._create_ds_worker_config(), vllm_config=vllm_config, ) @@ -292,7 +325,7 @@ def compute_log_prob(self, payload: dict): def update_actor(self, payload: dict): payload["processing"] = { - "post": ["apply_temperature", "compute_logprobs", "compute_entropy"], + "post": ["apply_temperature", "compute_logprobs", "compute_entropy"], #"loss_fn": "verl_grpo" "loss_fn": "grpo" } From 7a8b8253d1dc75e531ca080b7e1e61a9cbea8c84 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 16 Apr 2026 20:06:50 +0000 Subject: [PATCH 39/58] Fix batch_size to include rollout_n --- .../run_arctic_gsm8k_grpo_zorro_no.sh | 9 ++-- .../run_arctic_gsm8k_grpo_zorro_yes.sh | 5 +- examples/arctic_rl/run_gsm8k_grpo.sh | 54 +++++++++---------- verl/trainer/ppo/arctic_rl_client.py | 2 +- 4 files changed, 31 insertions(+), 39 deletions(-) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh index fb2f6d30e88..5f676618f2d 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh @@ -12,7 +12,6 @@ export USE_ARCTIC_ZORRO=0 export CUDA_VISIBLE_DEVICES= # BSZ=1024 -# MBS=256 # UBS=32 # ROLL_N=5 # MAX_STEPS=100 @@ -20,7 +19,6 @@ export CUDA_VISIBLE_DEVICES= # RESPONSE_LENGTH=1024 # BSZ=4 -# MBS=4 # UBS=2 # ROLL_N=5 # MAX_STEPS=4 @@ -28,7 +26,6 @@ export CUDA_VISIBLE_DEVICES= # RESPONSE_LENGTH=1024 BSZ=2 -MBS=2 UBS=2 ROLL_N=4 MAX_STEPS=4 @@ -40,8 +37,8 @@ LR=1e-6 LOGGER=console # LOGGER="['console','wandb']" -# USE_KL_LOSS=True -USE_KL_LOSS=False +USE_KL_LOSS=True +# USE_KL_LOSS=False # REMOVE_PADDING=True REMOVE_PADDING=False MODEL="Qwen/Qwen3-0.6B" @@ -83,7 +80,7 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.model.path=$MODEL \ actor_rollout_ref.actor.optim.lr=$LR \ actor_rollout_ref.model.use_remove_padding=$REMOVE_PADDING \ - actor_rollout_ref.actor.ppo_mini_batch_size=$MBS \ + actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=$UBS \ actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ actor_rollout_ref.actor.kl_loss_coef=0.001 \ diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh index b8e04007738..09ebc23d50d 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh @@ -13,7 +13,6 @@ export CUDA_VISIBLE_DEVICES= # BSZ=1024 BSZ=4 -MBS=4 UBS=2 ROLL_N=2 MAX_STEPS=4 @@ -21,7 +20,6 @@ PROMPT_LENGTH=64 RESPONSE_LENGTH=512 # BSZ=1 -# MBS=1 # UBS=1 # ROLL_N=16 # MAX_STEPS=4 @@ -29,7 +27,6 @@ RESPONSE_LENGTH=512 # RESPONSE_LENGTH=2048 # BSZ=2 -# MBS=2 # UBS=2 # ROLL_N=4 # MAX_STEPS=4 @@ -83,7 +80,7 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.model.path=$MODEL \ actor_rollout_ref.actor.optim.lr=$LR \ actor_rollout_ref.model.use_remove_padding=$REMOVE_PADDING \ - actor_rollout_ref.actor.ppo_mini_batch_size=$MBS \ + actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=$UBS \ actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ actor_rollout_ref.actor.kl_loss_coef=0.001 \ diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh index 3abfc33c7ae..0b8f615cfbb 100755 --- a/examples/arctic_rl/run_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_gsm8k_grpo.sh @@ -6,39 +6,37 @@ export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 # BSZ=1024 -# MBS=256 # UBS=32 ROLL_N=5 # MAX_STEPS=100 PROMPT_LENGTH=512 RESPONSE_LENGTH=1024 BSZ=8 -MBS=8 UBS=2 # ROLL_N=2 MAX_STEPS=4 # PROMPT_LENGTH=64 # RESPONSE_LENGTH=512 -# LR=0 +# LR=0 LR=1e-6 # LOGGER=console LOGGER="['console','wandb']" # USE_KL_LOSS=True -USE_KL_LOSS=False +USE_KL_LOSS=False # REMOVE_PADDING=True REMOVE_PADDING=False MODEL="Qwen/Qwen3-0.6B" # STRATEGY="fsdp" STRATEGY="fsdp2" -PYTHONUNBUFFERED=1 +PYTHONUNBUFFERED=1 HYDRA_FULL_ERROR=1 USE_LEGACY_WORKER_IMPL=disable NGPU_PER_NODE=4 ROLLOUT_NAME=vllm -experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_baseline" +experiment_name="qwen3-0.6B_ngpu$NGPU_PER_NODE_gbs$BSZ_rolln$ROLL_N_baseline" gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) if [[ $gpu_name == *"H200"* ]]; then echo "Running on Hopper" @@ -54,9 +52,9 @@ python3 -m verl.trainer.main_ppo \ algorithm.adv_estimator=grpo \ data.train_files=/code/shared/gsm8k/train.parquet \ data.val_files=/code/shared/gsm8k/test.parquet \ - data.train_batch_size=${BSZ} \ - data.max_prompt_length=${PROMPT_LENGTH} \ - data.max_response_length=${RESPONSE_LENGTH} \ + data.train_batch_size=$BSZ \ + data.max_prompt_length=$PROMPT_LENGTH \ + data.max_response_length=$RESPONSE_LENGTH \ data.filter_overlong_prompts=True \ data.truncation='error' \ data.shuffle=False \ @@ -64,41 +62,41 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.data_loader_seed=42 \ reward.num_workers=1 \ actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.model.path=${MODEL} \ - actor_rollout_ref.actor.optim.lr=${LR} \ - actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ - actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ + actor_rollout_ref.model.path=$MODEL \ + actor_rollout_ref.actor.optim.lr=$LR \ + actor_rollout_ref.model.use_remove_padding=$REMOVE_PADDING \ + actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=$UBS \ + actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ actor_rollout_ref.actor.kl_loss_coef=0.001 \ actor_rollout_ref.actor.kl_loss_type=low_var_kl \ actor_rollout_ref.actor.entropy_coeff=0 \ actor_rollout_ref.model.enable_gradient_checkpointing=True \ +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.actor.strategy=${STRATEGY} \ + actor_rollout_ref.actor.strategy=$STRATEGY \ actor_rollout_ref.actor.fsdp_config.param_offload=False \ actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=$UBS \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ + actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ actor_rollout_ref.rollout.enforce_eager=True \ - actor_rollout_ref.rollout.n=${ROLL_N} \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ + actor_rollout_ref.rollout.n=$ROLL_N \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=$UBS \ actor_rollout_ref.ref.fsdp_config.param_offload=False \ - actor_rollout_ref.ref.strategy=${STRATEGY} \ + actor_rollout_ref.ref.strategy=$STRATEGY \ algorithm.use_kl_in_reward=False \ - trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ + trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ trainer.critic_warmup=0 \ - trainer.logger=${LOGGER} \ - trainer.experiment_name=${experiment_name} \ + trainer.logger=$LOGGER \ + trainer.experiment_name=$experiment_name \ trainer.project_name='verl_arctic_grpo_gsm8k' \ trainer.val_before_train=False \ - trainer.n_gpus_per_node=${NGPU_PER_NODE} \ + trainer.n_gpus_per_node=$NGPU_PER_NODE \ trainer.nnodes=1 \ trainer.save_freq=-1 \ trainer.test_freq=-1 \ - trainer.total_training_steps=${MAX_STEPS} \ - trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log + trainer.total_training_steps=$MAX_STEPS \ + trainer.total_epochs=15 $@ 2>&1 | tee $experiment_name.log - # trainer.total_training_steps=${MAX_STEPS} \ + # trainer.total_training_steps=$MAX_STEPS \ diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 2ac27862558..82b59407d59 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -197,7 +197,7 @@ def _create_ds_config(self, n_gpus: int) -> dict[str, Any]: data_cfg = self.config.data micro_batch_size = actor_cfg.ppo_micro_batch_size_per_gpu or 1 - train_batch_size = data_cfg.train_batch_size + train_batch_size = data_cfg.train_batch_size * self.config.actor_rollout_ref.rollout.n grad_accum_steps = max(1, train_batch_size // (micro_batch_size * n_gpus)) train_seq_parallel_size = actor_cfg.fsdp_config.get("ulysses_sequence_parallel_size", 1) return { From ac63345287a6e440a9d2a106bf7173ddbe6d545b Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 16 Apr 2026 20:14:56 +0000 Subject: [PATCH 40/58] use ARL by default now Signed-off-by: Stas Bekman --- examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh | 1 + examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh index 5f676618f2d..73bbdbcc2f5 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh @@ -6,6 +6,7 @@ export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 export HF_HUB_OFFLINE=1 +export USE_ARCTIC_TRAINING_CLIENT=1 export USE_ARCTIC_ZORRO=0 # we want to make sure this runs on non-gpu client diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh index 09ebc23d50d..acbb27a9acf 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh @@ -6,6 +6,7 @@ export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 export HF_HUB_OFFLINE=1 +export USE_ARCTIC_TRAINING_CLIENT=1 export USE_ARCTIC_ZORRO=1 # we want to make sure this runs on non-gpu client From 9a60b7a513a750869bec700e858b62e661e817d4 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 16 Apr 2026 22:00:34 +0000 Subject: [PATCH 41/58] Async generate --- examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh | 2 +- verl/trainer/ppo/arctic_rl_client.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh index 73bbdbcc2f5..88462ce5c3d 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh @@ -77,7 +77,7 @@ python3 -m verl.trainer.main_ppo \ +data.seed=42 \ actor_rollout_ref.actor.data_loader_seed=42 \ reward.num_workers=1 \ - actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.rollout.agent.num_workers=4 \ actor_rollout_ref.model.path=$MODEL \ actor_rollout_ref.actor.optim.lr=$LR \ actor_rollout_ref.model.use_remove_padding=$REMOVE_PADDING \ diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 82b59407d59..f77062c745a 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -302,10 +302,10 @@ def initialize(self, model_name: str): "max_tokens": 1024, } - def generate(self, prompt_ids, sampling_params) -> list: + async def generate(self, prompt_ids, sampling_params) -> list: prompts = [self.tokenizer.decode(prompt_ids)] # TODO: pass prompt_ids directly merged_params = {**self._default_sampling_params, **sampling_params} - return self._client.generate(prompts=prompts, sampling_params=merged_params) + return await self._client.async_generate(prompts=prompts, sampling_params=merged_params) def compute_ref_log_prob(self, payload: dict): From 35ea0a5435e198a4d7123976e22208e0e7eb377a Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Fri, 17 Apr 2026 18:43:09 +0000 Subject: [PATCH 42/58] Gas support --- .../arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh | 15 ++++++++------- .../arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh | 10 +++++----- .../run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh | 5 +++-- .../run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh | 6 +++--- verl/trainer/ppo/arctic_rl_client.py | 2 +- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh index 88462ce5c3d..098169ef59e 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh @@ -6,9 +6,8 @@ export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 export HF_HUB_OFFLINE=1 +export HF_HOME=/checkpoint/huggingface export USE_ARCTIC_TRAINING_CLIENT=1 -export USE_ARCTIC_ZORRO=0 - # we want to make sure this runs on non-gpu client export CUDA_VISIBLE_DEVICES= @@ -26,12 +25,12 @@ export CUDA_VISIBLE_DEVICES= # PROMPT_LENGTH=512 # RESPONSE_LENGTH=1024 -BSZ=2 +BSZ=4 UBS=2 -ROLL_N=4 -MAX_STEPS=4 +ROLL_N=2 +MAX_STEPS=1 PROMPT_LENGTH=64 -RESPONSE_LENGTH=512 +RESPONSE_LENGTH=16 # LR=0 LR=1e-6 @@ -49,9 +48,10 @@ USE_LEGACY_WORKER_IMPL=disable NGPU_PER_NODE=1 ROLLOUT_NAME=arctic # entry point into ArcticRL USE_ARCTIC_RL=True +USE_ARCTIC_ZORRO=False # COLOCATE=True COLOCATE=False -experiment_name="qwen3-0.6B_ngpu$NGPU_PER_NODE_gbs$BSZ_rolln$ROLL_N_zorro$USE_ARCTIC_ZORRO" +experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_zorro${USE_ARCTIC_ZORRO}" gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) if [[ $gpu_name == *"H200"* ]]; then @@ -108,6 +108,7 @@ python3 -m verl.trainer.main_ppo \ arctic_rl.training_gpus=1\ arctic_rl.sampling_gpus=2\ arctic_rl.log_prob_gpus=1\ + arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ trainer.critic_warmup=0 \ trainer.logger=$LOGGER \ trainer.experiment_name=$experiment_name \ diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh index acbb27a9acf..4fbcf45e11a 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh @@ -6,9 +6,8 @@ export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 export HF_HUB_OFFLINE=1 +export HF_HOME=/checkpoint/huggingface export USE_ARCTIC_TRAINING_CLIENT=1 -export USE_ARCTIC_ZORRO=1 - # we want to make sure this runs on non-gpu client export CUDA_VISIBLE_DEVICES= @@ -50,8 +49,9 @@ USE_LEGACY_WORKER_IMPL=disable NGPU_PER_NODE=1 ROLLOUT_NAME=arctic # entry point into ArcticRL USE_ARCTIC_RL=True +USE_ARCTIC_ZORRO=True COLOCATE=False -experiment_name="qwen3-0.6B_ngpu$NGPU_PER_NODE_gbs$BSZ_rolln$ROLL_N_zorro$USE_ARCTIC_ZORRO" +experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_zorro${USE_ARCTIC_ZORRO}" gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) if [[ $gpu_name == *"H200"* ]]; then @@ -77,7 +77,7 @@ python3 -m verl.trainer.main_ppo \ +data.seed=42 \ actor_rollout_ref.actor.data_loader_seed=42 \ reward.num_workers=1 \ - actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.rollout.agent.num_workers=4 \ actor_rollout_ref.model.path=$MODEL \ actor_rollout_ref.actor.optim.lr=$LR \ actor_rollout_ref.model.use_remove_padding=$REMOVE_PADDING \ @@ -108,7 +108,7 @@ python3 -m verl.trainer.main_ppo \ arctic_rl.training_gpus=1\ arctic_rl.sampling_gpus=2\ arctic_rl.log_prob_gpus=1\ - arctic_rl.use_zorro=True \ + arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ trainer.critic_warmup=0 \ trainer.logger=$LOGGER \ trainer.experiment_name=$experiment_name \ diff --git a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh index 9cc1bfa02b9..94a2ab25886 100755 --- a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh +++ b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh @@ -21,12 +21,13 @@ export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 export HF_HUB_OFFLINE=1 - +export HF_HOME=/checkpoint/huggingface +export USE_ARCTIC_TRAINING_CLIENT=1 export CUDA_VISIBLE_DEVICES= USE_ARCTIC_RL=True # entry point into ArcticRL USE_LEGACY_WORKER_IMPL=disable -ROLLOUT_NAME=arctic +ROLLOUT_NAME=arctic NUM_AGENT_WORKERS=1 NGPU_PER_NODE=1 diff --git a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh index b1382390ccb..6bc994c915a 100755 --- a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh +++ b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh @@ -21,13 +21,13 @@ export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 export HF_HUB_OFFLINE=1 -export USE_ARCTIC_ZORRO=1 - +export HF_HOME=/checkpoint/huggingface +export USE_ARCTIC_TRAINING_CLIENT=1 export CUDA_VISIBLE_DEVICES= USE_ARCTIC_RL=True # entry point into ArcticRL USE_LEGACY_WORKER_IMPL=disable -ROLLOUT_NAME=arctic +ROLLOUT_NAME=arctic NUM_AGENT_WORKERS=1 NGPU_PER_NODE=1 diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index f77062c745a..016697dd86f 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -326,7 +326,7 @@ def compute_log_prob(self, payload: dict): def update_actor(self, payload: dict): payload["processing"] = { "post": ["apply_temperature", "compute_logprobs", "compute_entropy"], - #"loss_fn": "verl_grpo" + # "loss_fn": "verl_grpo" "loss_fn": "grpo" } def _left_pad(t: torch.Tensor, seq_len: int) -> torch.Tensor: From 5ce5fba67329cb25ac0a38a1c5d7316a7f07e70f Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Sat, 18 Apr 2026 00:06:19 +0000 Subject: [PATCH 43/58] new launchers and some small cleanup in the old ones Signed-off-by: Stas Bekman --- .../run_arctic_gsm8k_grpo_zorro_yes.sh | 23 ++- ...un_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh | 6 +- .../arctic_rl/run_bird_grpo_arl_zorro_yes.sh | 147 ++++++++++++++++++ examples/arctic_rl/run_bird_grpo_baseline.sh | 134 ++++++++++++++++ examples/arctic_rl/run_gsm8k_grpo.sh | 1 + 5 files changed, 297 insertions(+), 14 deletions(-) create mode 100755 examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh create mode 100755 examples/arctic_rl/run_bird_grpo_baseline.sh diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh index 4fbcf45e11a..892cef3e141 100755 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh +++ b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh @@ -11,13 +11,12 @@ export USE_ARCTIC_TRAINING_CLIENT=1 # we want to make sure this runs on non-gpu client export CUDA_VISIBLE_DEVICES= -# BSZ=1024 -BSZ=4 -UBS=2 -ROLL_N=2 -MAX_STEPS=4 -PROMPT_LENGTH=64 -RESPONSE_LENGTH=512 +BSZ=16 +UBS=16 +ROLL_N=5 +MAX_STEPS=40 +PROMPT_LENGTH=512 +RESPONSE_LENGTH=1024 # BSZ=1 # UBS=1 @@ -36,8 +35,8 @@ RESPONSE_LENGTH=512 # LR=0 LR=1e-6 -LOGGER=console -# LOGGER="['console','wandb']" +#LOGGER=console +LOGGER="['console','wandb']" # USE_KL_LOSS=True USE_KL_LOSS=False # REMOVE_PADDING=True @@ -112,13 +111,13 @@ python3 -m verl.trainer.main_ppo \ trainer.critic_warmup=0 \ trainer.logger=$LOGGER \ trainer.experiment_name=$experiment_name \ - trainer.project_name='verl_arctic_grpo_gsm8k' \ + trainer.project_name=arctic_rl_bird_sql \ trainer.val_before_train=False \ trainer.n_gpus_per_node=$NGPU_PER_NODE \ trainer.nnodes=1 \ trainer.save_freq=-1 \ trainer.test_freq=-1 \ trainer.total_training_steps=$MAX_STEPS \ - trainer.total_epochs=15 $@ 2>&1 | tee $experiment_name.log + trainer.total_epochs=15 \ + "$@" 2>&1 | tee $experiment_name.log - # trainer.total_training_steps=$MAX_STEPS \ diff --git a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh index 94a2ab25886..39462bb84bc 100755 --- a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh +++ b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh @@ -36,10 +36,12 @@ NGPU_PER_NODE=1 # RESPONSE_LEN=4096 # ROLL_N=16 -BSZ=2 +BSZ=16 +UBS=4 PROMPT_LEN=16384 RESPONSE_LEN=4096 -ROLL_N=2 +ROLL_N=16 + # LOGGER=console LOGGER="['console','wandb']" diff --git a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh new file mode 100755 index 00000000000..cee1ebbd36d --- /dev/null +++ b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# GRPO training for Qwen3-1.7B on BIRD SQL dataset +# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) +# +# 1 node, 8 GPUs +# +# Prerequisites: +# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py +# 2. pip install func_timeout + +set -x + +SCRIPT_DIR="$(cd "$(dirname "$BASH_SOURCE[0]")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export PYTHONPATH="$REPO_ROOT:$PYTHONPATH" + +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +export HF_HUB_OFFLINE=1 +export HF_HOME=/checkpoint/huggingface +export USE_ARCTIC_TRAINING_CLIENT=1 # entry point into ArcticRL +export CUDA_VISIBLE_DEVICES= + +USE_LEGACY_WORKER_IMPL=disable +ROLLOUT_NAME=arctic +NGPU_PER_NODE=1 + +USE_ARCTIC_RL=True +USE_ARCTIC_ZORRO=True +COLOCATE=False + +BSZ=32 +ROLL_N=16 +MAX_STEPS=10 +PROMPT_LEN=16384 +RESPONSE_LEN=4096 + +#LOGGER=console +LOGGER="['console','wandb']" + +#MODEL_SHORT=Qwen3-1.7B +MODEL_SHORT=Qwen3-0.6B + +MODEL=Qwen/$MODEL_SHORT + +experiment_name="bird_grpo_${MODEL_SHORT}_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_arl_zorro_yes" + +gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) +if [[ $gpu_name == *"H200"* ]]; then + echo "Running on Hopper" + flash_attention_v=flash_attention_3 +elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then + echo "Running on Blackwell" + flash_attention_v=flash_attention_2 +else + echo "Running on unknown: $gpu_name; don't know which FA version to use" +fi + +# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" +# TRAIN_FILES="$DATA_DIR/train.parquet" +# VAL_FILES="$DATA_DIR/val.parquet" + + +DATA_DIR="/code/shared/open-source-text2sql" +#TRAIN_FILES="$DATA_DIR/train.parquet" +#TRAIN_FILES="$DATA_DIR/train-1000.parquet" +TRAIN_FILES="$DATA_DIR/train-100.parquet" +VAL_FILES="$DATA_DIR/val.parquet" + +# LOG_PROBS=True +LOG_PROBS=False + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + algorithm.norm_adv_by_std_in_grpo=True \ + algorithm.use_kl_in_reward=False \ + algorithm.kl_ctrl.kl_coef=0.001 \ + data.train_files=$TRAIN_FILES \ + data.val_files=$VAL_FILES \ + data.train_batch_size=$BSZ \ + data.max_prompt_length=$PROMPT_LEN \ + data.max_response_length=$RESPONSE_LEN \ + data.filter_overlong_prompts=True \ + data.filter_overlong_prompts_workers=1 \ + data.truncation=left \ + actor_rollout_ref.model.path=$MODEL \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ + actor_rollout_ref.model.use_liger=True \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.actor.use_torch_compile=True \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.lr=5e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ + actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=$ROLL_N \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.top_p=1.0 \ + actor_rollout_ref.rollout.calculate_log_probs=$LOG_PROBS \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.max_num_seqs=256 \ + actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ + actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ + actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ + actor_rollout_ref.rollout.val_kwargs.temperature=0 \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.nccl_timeout=1800 \ + trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ + trainer.use_arctic_rl=$USE_ARCTIC_RL \ + trainer.balance_batch=False \ + trainer.default_local_dir=/data-fast/sql-rl/$experiment_name \ + trainer.logger=$LOGGER \ + trainer.project_name=arctic_rl_bird_sql \ + trainer.experiment_name=$experiment_name \ + trainer.n_gpus_per_node=$NGPU_PER_NODE \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=10 \ + trainer.val_before_train=False \ + custom_reward_function.path="$SCRIPT_DIR/bird_reward.py" \ + custom_reward_function.name=compute_score \ + trainer.total_training_steps=$MAX_STEPS \ + arctic_rl.colocate=$COLOCATE \ + arctic_rl.training_gpus=1\ + arctic_rl.sampling_gpus=2\ + arctic_rl.log_prob_gpus=1\ + arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ + "$@" 2>&1 | tee $experiment_name.log diff --git a/examples/arctic_rl/run_bird_grpo_baseline.sh b/examples/arctic_rl/run_bird_grpo_baseline.sh new file mode 100755 index 00000000000..8d160368473 --- /dev/null +++ b/examples/arctic_rl/run_bird_grpo_baseline.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# GRPO training for Qwen3-1.7B on BIRD SQL dataset +# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) +# +# 1 node, 8 GPUs +# +# Prerequisites: +# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py +# 2. pip install func_timeout + +set -x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export PYTHONPATH="$REPO_ROOT:$PYTHONPATH" +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +export HF_HUB_OFFLINE=1 +export HF_HOME=/checkpoint/huggingface + +USE_LEGACY_WORKER_IMPL=disable +ROLLOUT_NAME=vllm +NGPU_PER_NODE=1 + +BSZ=32 +ROLL_N=16 +MAX_STEPS=10 +PROMPT_LEN=16384 +RESPONSE_LEN=4096 + +#LOGGER=console +LOGGER="['console','wandb']" + +#MODEL_SHORT=Qwen3-1.7B +MODEL_SHORT=Qwen3-0.6B + +MODEL=Qwen/$MODEL_SHORT + +experiment_name="bird_grpo_${MODEL_SHORT}_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_baseline" + +gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) +if [[ $gpu_name == *"H200"* ]]; then + echo "Running on Hopper" + flash_attention_v=flash_attention_3 +elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then + echo "Running on Blackwell" + flash_attention_v=flash_attention_2 +else + echo "Running on unknown: $gpu_name; don't know which FA version to use" +fi + +# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" +# TRAIN_FILES="$DATA_DIR/train.parquet" +# VAL_FILES="$DATA_DIR/val.parquet" + + +DATA_DIR="/code/shared/open-source-text2sql" +#TRAIN_FILES="$DATA_DIR/train.parquet" +#TRAIN_FILES="$DATA_DIR/train-1000.parquet" +TRAIN_FILES="$DATA_DIR/train-100.parquet" +VAL_FILES="$DATA_DIR/val.parquet" + +# LOG_PROBS=True +LOG_PROBS=False + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + algorithm.norm_adv_by_std_in_grpo=True \ + algorithm.use_kl_in_reward=False \ + algorithm.kl_ctrl.kl_coef=0.001 \ + data.train_files=$TRAIN_FILES \ + data.val_files=$VAL_FILES \ + data.train_batch_size=$BSZ \ + data.max_prompt_length=$PROMPT_LEN \ + data.max_response_length=$RESPONSE_LEN \ + data.filter_overlong_prompts=True \ + data.filter_overlong_prompts_workers=1 \ + data.truncation=left \ + actor_rollout_ref.model.path=$MODEL \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ + actor_rollout_ref.model.use_liger=True \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.actor.use_torch_compile=True \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.lr=5e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ + actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=$ROLL_N \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.top_p=1.0 \ + actor_rollout_ref.rollout.calculate_log_probs=$LOG_PROBS \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.max_num_seqs=256 \ + actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ + actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ + actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ + actor_rollout_ref.rollout.val_kwargs.temperature=0 \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.nccl_timeout=1800 \ + trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ + trainer.balance_batch=False \ + trainer.default_local_dir=/data-fast/sql-rl/$experiment_name \ + trainer.logger=$LOGGER \ + trainer.project_name=arctic_rl_bird_sql \ + trainer.experiment_name=$experiment_name \ + trainer.n_gpus_per_node=$NGPU_PER_NODE \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=10 \ + trainer.val_before_train=False \ + custom_reward_function.path="$SCRIPT_DIR/bird_reward.py" \ + custom_reward_function.name=compute_score \ + trainer.total_training_steps=$MAX_STEPS \ + "$@" 2>&1 | tee $experiment_name.log diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh index 0b8f615cfbb..3e4d53831f1 100755 --- a/examples/arctic_rl/run_gsm8k_grpo.sh +++ b/examples/arctic_rl/run_gsm8k_grpo.sh @@ -5,6 +5,7 @@ set -x export PYTHONUNBUFFERED=1 export HYDRA_FULL_ERROR=1 export RAY_DEDUP_LOGS=0 + # BSZ=1024 # UBS=32 ROLL_N=5 From 48ac315358277f15c72928f5888db1993e627bc2 Mon Sep 17 00:00:00 2001 From: tunji-ruwase_snow Date: Tue, 21 Apr 2026 15:51:06 +0000 Subject: [PATCH 44/58] Cleanup; arl yaml config --- .../arctic_rl/run_bird_grpo_arl_zorro_yes.sh | 12 +- verl/trainer/config/ppo_trainer.yaml | 2 + verl/trainer/ppo/arctic_rl_client.py | 2 +- verl/workers/arctic_workers.py | 107 +++++++++++++++--- verl/workers/utils/padding.py | 78 ------------- 5 files changed, 100 insertions(+), 101 deletions(-) diff --git a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh index cee1ebbd36d..66485146394 100755 --- a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh +++ b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh @@ -31,14 +31,16 @@ USE_ARCTIC_RL=True USE_ARCTIC_ZORRO=True COLOCATE=False -BSZ=32 -ROLL_N=16 +# BSZ=32 +# ROLL_N=16 +BSZ=4 +ROLL_N=2 MAX_STEPS=10 PROMPT_LEN=16384 RESPONSE_LEN=4096 -#LOGGER=console -LOGGER="['console','wandb']" +LOGGER=console +# LOGGER="['console','wandb']" #MODEL_SHORT=Qwen3-1.7B MODEL_SHORT=Qwen3-0.6B @@ -141,7 +143,7 @@ python3 -m verl.trainer.main_ppo \ trainer.total_training_steps=$MAX_STEPS \ arctic_rl.colocate=$COLOCATE \ arctic_rl.training_gpus=1\ - arctic_rl.sampling_gpus=2\ + arctic_rl.sampling_gpus=1\ arctic_rl.log_prob_gpus=1\ arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ "$@" 2>&1 | tee $experiment_name.log diff --git a/verl/trainer/config/ppo_trainer.yaml b/verl/trainer/config/ppo_trainer.yaml index e2908368851..bbf27799c39 100644 --- a/verl/trainer/config/ppo_trainer.yaml +++ b/verl/trainer/config/ppo_trainer.yaml @@ -321,6 +321,8 @@ arctic_rl: # whether to use colocate mode colocate: False + use_arctic_rl_client: True + training_gpus: 1 sampling_gpus: 1 log_prob_gpus: 1 diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 016697dd86f..1c7fc414804 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -14,7 +14,7 @@ def create_arctic_rl_client(config): - cls = ArcticRLClientWrapper if USE_ARCTIC_TRAINING_CLIENT else ArcticRLClient4VeRL + cls = ArcticRLClientWrapper if config.arctic_rl.use_arctic_rl_client else ArcticRLClient4VeRL sched_pg = placement_group([{"GPU": 0, "CPU": 1}]) return ray.remote( num_cpus=0, diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 88032b3b16e..db2527bf83e 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -88,6 +88,82 @@ def load_dump_data(train_batch_size, roll_n) -> dict[str, DataProto]: return dump_data + +def no_padding_2_padding_prompt_response(tensor: torch.Tensor, data: TensorDict, pad_token_id) -> torch.Tensor: + """Convert jagged tensor into a left padded prompt and right padded prompt of [bsz, max_response_len], which looks like + tensor([ + [pad...prompt | response...pad], + [pad...prompt | response...pad], + [pad...prompt | response...pad] + ]) + + Args: + tensor: a nested tensor or a 1D tensor in shape (total_nnz,), + total_nnz is the total number of tokens across all sequences in the batch + data: TensorDict with "prompts", "responses", "attention_mask" + pad_token_id: token to pad with + + Returns: + tensor: sliced prompt+response tensor of shape [bsz, max_response_len] w/ left and right padding + + """ + # print(f"{tensor.is_nested=}") + values = tensor.values() if tensor.is_nested else tensor + prompt_ids = data["prompts"] + response_ids = data["responses"] + attention_mask = data["attention_mask"] + # print(f"{prompt_ids.shape=}") + # print(f"{response_ids.shape=}") + # print(f"{attention_mask.shape=}") + # print(f"{attention_mask=}") + + max_prompt_len = tu.get_non_tensor_data(data=data, key="max_prompt_len", default=-1) + max_response_len = tu.get_non_tensor_data(data=data, key="max_response_len", default=-1) + # print(f"data {max_prompt_len=}") + # print(f"data {max_response_len=}") + + # print(f"{prompt_ids.is_nested=}") + if prompt_ids.is_nested: + prompt_lens = prompt_ids.offsets().diff() + response_lens = response_ids.offsets().diff() + if max_prompt_len < 0: + max_prompt_len = prompt_lens.max().item() + if max_response_len < 0: + max_response_len = response_lens.max().item() + else: + assert not attention_mask.is_nested + prompt_lens = attention_mask[:, : prompt_ids.shape[1]].sum(dim=1) + response_lens = attention_mask[:, prompt_ids.shape[1] :].sum(dim=1) + max_prompt_len = prompt_ids.shape[1] + max_response_len = response_ids.shape[1] + + sequence_lens = prompt_lens + response_lens + sequence_offsets = sequence_lens.cumsum(dim=0) + # print(f"{data=}") + # print(f"{prompt_lens=}") + # print(f"{response_lens=}") + # print(f"{max_prompt_len=}") + # print(f"{max_response_len=}") + # print(f"{sequence_offsets=}") + # print(f"{values=}") + # print(f"{values.shape=}") + assert sequence_offsets[-1].item() == values.shape[0], f"{sequence_offsets[-1].item()} != {values.shape[0]}" + + input_ids_list = [] + for prompt_len, resp_len, seq_offset in zip(prompt_lens, response_lens, sequence_offsets, strict=True): + prompt_pad_size = max_prompt_len - prompt_len + response_pad_size = max_response_len - resp_len + prompt = values[seq_offset - prompt_len - resp_len: seq_offset - resp_len] + response = values[seq_offset - resp_len: seq_offset] + prompt_padded_left = F.pad(prompt, (prompt_pad_size, 0), value=pad_token_id) + response_padded_right = F.pad(response, (0, response_pad_size), value=pad_token_id) + input_ids_list.append(torch.cat((prompt_padded_left, response_padded_right))) + + output = torch.stack(input_ids_list, dim=0) + #print(f"{output=}") + return output, max_prompt_len, max_response_len + + def prepare_model_inputs_remove_padding(micro_batch: TensorDict): from verl.utils import tensordict_utils as tu from verl.utils.dataset.dataset_utils import DatasetPadMode @@ -185,7 +261,6 @@ def prepare_padded_dss_batch_dict(data: TensorDict, pad_token_id) -> dict: input_ids = data['input_ids'] position_ids = data['position_ids'] - from verl.workers.utils.padding import no_padding_2_padding_prompt_response orig_iput_ids_shape = input_ids.shape orig_position_ids_shape = position_ids.shape input_ids, max_prompt_len, max_response_len = no_padding_2_padding_prompt_response(tensor=input_ids, data=data, pad_token_id=pad_token_id) @@ -193,7 +268,7 @@ def prepare_padded_dss_batch_dict(data: TensorDict, pad_token_id) -> dict: position_ids, _, _= no_padding_2_padding_prompt_response(tensor=position_ids, data=data, pad_token_id=0) attention_mask = data['attention_mask'] - print(f"{input_ids.shape=} {position_ids.shape=} {attention_mask.shape=} {orig_iput_ids_shape=} {orig_position_ids_shape=}") + # print(f"{input_ids.shape=} {position_ids.shape=} {attention_mask.shape=} {orig_iput_ids_shape=} {orig_position_ids_shape=}") dss_batch_dict = dict( input_ids=input_ids, @@ -420,25 +495,23 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: position_ids = data['position_ids'] #input_ids = input_ids.unbind() - from verl.workers.utils.padding import no_padding_2_padding_prompt_response # XXX: move to init - input_ids, max_prompt_len, max_response_len = no_padding_2_padding_prompt_response(tensor=input_ids, data=data, pad_token_id=self.pad_token_id) # XXX: 0 pad on pos ids is odd, check the original - perhaps need to re-build pos ids? position_ids, _, _= no_padding_2_padding_prompt_response(tensor=position_ids, data=data, pad_token_id=0) - print(f"{input_ids.shape=}") - print(f"{input_ids=}") + # print(f"{input_ids.shape=}") + # print(f"{input_ids=}") #input_ids = torch.nested.to_padded_tensor(input_ids, padding=4.2) #position_ids = torch.nested.to_padded_tensor(position_ids, padding=4.2) - print(f"{data['attention_mask'].shape=}") - print(f"{data['attention_mask']=}") - print(f"{input_ids.shape=}") - print(f"{input_ids=}") - print(f"{position_ids.shape=}") - print(f"{position_ids=}") + # print(f"{data['attention_mask'].shape=}") + # print(f"{data['attention_mask']=}") + # print(f"{input_ids.shape=}") + # print(f"{input_ids=}") + # print(f"{position_ids.shape=}") + # print(f"{position_ids=}") # XXX: fixme # batch = batch[0] @@ -498,7 +571,7 @@ def safe_serialize(obj): # print(f"update_actor: {loss=}") metrics = response['metrics'] loss = metrics.pop("loss") - print(f"update_actor: {metrics=}") + # print(f"update_actor: {metrics=}") from verl.utils.metric import AggregationType, Metric @@ -519,7 +592,7 @@ def safe_serialize(obj): # } # print(f"{data=}") - print(f"{data["input_ids"].shape=}") + # print(f"{data["input_ids"].shape=}") # expected output so far # @@ -699,7 +772,7 @@ def compute_any_log_prob(self, data: TensorDict, compute_log_prob_fn) -> TensorD response = ray.get(compute_log_prob_fn.remote(payload)) - print(f"compute_any_log_prob: {response['batch']['entropy'].shape=} {response['batch']['log_probs'].shape=}") + # print(f"compute_any_log_prob: {response['batch']['entropy'].shape=} {response['batch']['log_probs'].shape=}") #batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) #model_output = batch_output.pop("model_output", {}) @@ -707,10 +780,10 @@ def compute_any_log_prob(self, data: TensorDict, compute_log_prob_fn) -> TensorD # verl wants a full [bs, max_prompt_len+max_response_len] tensors and jagged entropy = prepand_max_prompt_len_zeros(response['batch']['entropy'], max_prompt_len) log_probs = prepand_max_prompt_len_zeros(response['batch']['log_probs'], max_prompt_len) - print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") + # print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") entropy = make_njt(data, entropy) log_probs = make_njt(data, log_probs) - print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") + # print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") model_output = dict(entropy=entropy, log_probs=log_probs) metrics = response['metrics'] diff --git a/verl/workers/utils/padding.py b/verl/workers/utils/padding.py index cafa0d310a1..d8e53e40c7b 100644 --- a/verl/workers/utils/padding.py +++ b/verl/workers/utils/padding.py @@ -138,81 +138,3 @@ def no_padding_2_padding(tensor: torch.Tensor, data: TensorDict) -> torch.Tensor output = torch.stack(response_list, dim=0) return output - - - -def no_padding_2_padding_prompt_response(tensor: torch.Tensor, data: TensorDict, pad_token_id) -> torch.Tensor: - """Convert jagged tensor into a left padded prompt and right padded prompt of [bsz, max_response_len], which looks like - tensor([ - [pad...prompt | response...pad], - [pad...prompt | response...pad], - [pad...prompt | response...pad] - ]) - - Args: - tensor: a nested tensor or a 1D tensor in shape (total_nnz,), - total_nnz is the total number of tokens across all sequences in the batch - data: TensorDict with "prompts", "responses", "attention_mask" - pad_token_id: token to pad with - - Returns: - tensor: sliced prompt+response tensor of shape [bsz, max_response_len] w/ left and right padding - - """ - # print(f"{tensor.is_nested=}") - values = tensor.values() if tensor.is_nested else tensor - prompt_ids = data["prompts"] - response_ids = data["responses"] - attention_mask = data["attention_mask"] - # print(f"{prompt_ids.shape=}") - # print(f"{response_ids.shape=}") - # print(f"{attention_mask.shape=}") - # print(f"{attention_mask=}") - - max_prompt_len = tu.get_non_tensor_data(data=data, key="max_prompt_len", default=-1) - max_response_len = tu.get_non_tensor_data(data=data, key="max_response_len", default=-1) - print(f"data {max_prompt_len=}") - print(f"data {max_response_len=}") - - # print(f"{prompt_ids.is_nested=}") - if prompt_ids.is_nested: - prompt_lens = prompt_ids.offsets().diff() - response_lens = response_ids.offsets().diff() - if max_prompt_len < 0: - max_prompt_len = prompt_lens.max().item() - if max_response_len < 0: - max_response_len = response_lens.max().item() - else: - assert not attention_mask.is_nested - prompt_lens = attention_mask[:, : prompt_ids.shape[1]].sum(dim=1) - response_lens = attention_mask[:, prompt_ids.shape[1] :].sum(dim=1) - max_prompt_len = prompt_ids.shape[1] - max_response_len = response_ids.shape[1] - - sequence_lens = prompt_lens + response_lens - sequence_offsets = sequence_lens.cumsum(dim=0) - # print(f"{data=}") - # print(f"{prompt_lens=}") - # print(f"{response_lens=}") - # print(f"{max_prompt_len=}") - # print(f"{max_response_len=}") - # print(f"{sequence_offsets=}") - # print(f"{values=}") - # print(f"{values.shape=}") - assert sequence_offsets[-1].item() == values.shape[0], f"{sequence_offsets[-1].item()} != {values.shape[0]}" - - input_ids_list = [] - for prompt_len, resp_len, seq_offset in zip(prompt_lens, response_lens, sequence_offsets, strict=True): - prompt_pad_size = max_prompt_len - prompt_len - response_pad_size = max_response_len - resp_len - prompt = values[seq_offset - prompt_len - resp_len: seq_offset - resp_len] - response = values[seq_offset - resp_len: seq_offset] - prompt_padded_left = F.pad(prompt, (prompt_pad_size, 0), value=pad_token_id) - response_padded_right = F.pad(response, (0, response_pad_size), value=pad_token_id) - input_ids_list.append(torch.cat((prompt_padded_left, response_padded_right))) - - output = torch.stack(input_ids_list, dim=0) - #print(f"{output=}") - return output, max_prompt_len, max_response_len - - From 6f3713dab991a829e359799b552261c971c226b3 Mon Sep 17 00:00:00 2001 From: tunji-ruwase_snow Date: Tue, 21 Apr 2026 22:37:18 +0000 Subject: [PATCH 45/58] Remove temp arl client --- .../arctic_rl/run_bird_grpo_arl_zorro_yes.sh | 2 +- verl/trainer/config/ppo_trainer.yaml | 2 - verl/trainer/ppo/arctic_rl_client.py | 156 +----------------- .../rollout/arctic_rollout/arctic_rollout.py | 21 ++- 4 files changed, 13 insertions(+), 168 deletions(-) diff --git a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh index 66485146394..0eabeb33692 100755 --- a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh +++ b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh @@ -143,7 +143,7 @@ python3 -m verl.trainer.main_ppo \ trainer.total_training_steps=$MAX_STEPS \ arctic_rl.colocate=$COLOCATE \ arctic_rl.training_gpus=1\ - arctic_rl.sampling_gpus=1\ + arctic_rl.sampling_gpus=2\ arctic_rl.log_prob_gpus=1\ arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ "$@" 2>&1 | tee $experiment_name.log diff --git a/verl/trainer/config/ppo_trainer.yaml b/verl/trainer/config/ppo_trainer.yaml index bbf27799c39..e2908368851 100644 --- a/verl/trainer/config/ppo_trainer.yaml +++ b/verl/trainer/config/ppo_trainer.yaml @@ -321,8 +321,6 @@ arctic_rl: # whether to use colocate mode colocate: False - use_arctic_rl_client: True - training_gpus: 1 sampling_gpus: 1 log_prob_gpus: 1 diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index 1c7fc414804..c74fd669266 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -9,12 +9,7 @@ from ray.util.placement_group import placement_group from verl.workers.rollout.replica import TokenOutput -USE_ARCTIC_TRAINING_CLIENT = os.environ.get("USE_ARCTIC_TRAINING_CLIENT", "0") == "1" -USE_ARCTIC_ZORRO = os.environ.get("USE_ARCTIC_ZORRO", "0") == "1" - - def create_arctic_rl_client(config): - cls = ArcticRLClientWrapper if config.arctic_rl.use_arctic_rl_client else ArcticRLClient4VeRL sched_pg = placement_group([{"GPU": 0, "CPU": 1}]) return ray.remote( num_cpus=0, @@ -23,7 +18,7 @@ def create_arctic_rl_client(config): placement_group=sched_pg, placement_group_capture_child_tasks=True, ), - )(cls).remote(config) + )(ArcticRLClientWrapper).remote(config) def create_meta_model(name_or_path: str): model_config = AutoConfig.from_pretrained(name_or_path) @@ -32,155 +27,8 @@ def create_meta_model(name_or_path: str): return meta_model -class ArcticRLClient4VeRL: - def __init__(self, config): - """ - config: verl's full config - """ - self.config = config - self.use_zorro = self.config.arctic_rl.use_zorro - #print(f"ArcticRLClient4VeRL {config=}") - - self.arctic_inference_client = DSSInferenceClient(dss_server_url="http://localhost:7000") - self.arctic_training_client = DSSTrainingClient(dss_server_url="http://localhost:7000") - self.arctic_log_prob_client = DSSLogProbClient(dss_server_url="http://localhost:7000") - - - def is_zorro_enabled(self): - return self.use_zorro - - def initialize(self, model_name: str): - vllm_config = { - "temperature": 0.0, - "top_p": 1.0, - "top_k": 0, - "max_tokens": 1024, - "stop_sequences": [], - "stop_token_ids": [], - } - self.inference_engine = self.arctic_inference_client.initialize( - model_name=model_name, - vllm_config=vllm_config, - ) - self.log_prob_engine = self.arctic_log_prob_client.initialize( - model_name=model_name, - vllm_config=vllm_config, - ) - - ds_config = { - "train_micro_batch_size_per_gpu": 1, - "train_batch_size": 1, - "gradient_accumulation_steps": 1, - "sequence_parallel_size": 1, - "zero_optimization": { - "stage": 1, - }, - } - - # currently verl wants '+' before the setting, i.e. +actor_rollout_ref.model.override_config.attn_implementation=flash_attention_3 - attn_implementation = self.config.actor_rollout_ref.model.override_config.get('attn_implementation', 'eager') - if attn_implementation == "eager": - raise ValueError("set actor_rollout_ref.model.override_config.attn_implementation to some variant of flash attention") - - #attn_implementation="flash_attention_3" - - training_config = { - "optimizer": { - "lr": 0.0002, - "weight_decay": 0.0, - "betas": [0.9, 0.999], - }, - "lr_scheduler": {"warmup_ratio": 0.05}, - "training_horizon": 10, - "max_length": 8096, - "model_config": None, - "attn_implementation": attn_implementation, - } - - if self.is_zorro_enabled(): - # XXX: can't find where it's configured - use_unpad = True - - training_config.update( - use_zorro=True, - response_len=self.config.data.max_response_length, - max_token_len=self.config.actor_rollout_ref.rollout.max_num_batched_tokens, - rollout_n=self.config.actor_rollout_ref.rollout.n, - temperature=self.config.actor_rollout_ref.rollout.temperature, - use_unpad=use_unpad, - ) - #print(f"{training_config=}") - - self.training_engine = self.arctic_training_client.initialize( - model=create_meta_model(model_name), - ds_config=ds_config, - training_config=training_config) - - self.tokenizer = AutoTokenizer.from_pretrained(model_name) - - def generate(self, prompt_ids, sampling_params) -> TokenOutput: - prompts = [self.tokenizer.decode(prompt_ids)] - return self.inference_engine.generate( - prompts=prompts, - sampling_params=sampling_params, - ) - - - # TODO: this should use the reference engine instead of the training engine - def compute_ref_log_prob(self, payload: dict): - response = self.training_engine.fwd_no_grad(**payload) - # if entropy is not None: - # entropy = torch.tensor(entropy).squeeze() - # if log_probs is not None: - # log_probs = torch.tensor(log_probs).squeeze() - print(f"arctic_rl_client.compute_ref_log_prob: {response['batch']['entropy'].shape=}, {response['batch']['log_probs'].shape=}") - return response - - - def compute_log_prob(self, payload: dict): - # XXX: somehow we need to differentiate which model is this called on ref vs actor - at the moment it's always actor hardcoded - response = self.training_engine.fwd_no_grad(**payload) - - # XXX: for some reason no_padding_2_padding expects a 1D tensor - not sure how it'll work for - # bs>1 - # I think it may have to do with tensor.is_nested - different path/logic - # so most likely we need to convert these 2 into TensorDict - # if entropy is not None: - # entropy = torch.tensor(entropy).squeeze() - # if log_probs is not None: - # log_probs = torch.tensor(log_probs).squeeze() - print(f"arctic_rl_client.compute_log_prob: {response['batch']['entropy'].shape=}, {response['batch']['log_probs'].shape=}") - return response - - - def update_actor(self, payload: dict): - _ = self.training_engine.forward(**payload) - bwd_response = self.training_engine.backward() - step_response = self.training_engine.step() - - step_response["metrics"].update(**bwd_response["metrics"]) - - # metrics.update({"global_steps": [global_steps], "last_lr": [last_lr]}) - - # print(f"arctic_rl_client.update_actor: {loss=}") - # print(f"arctic_rl_client.update_actor: {metrics=}") - # return loss.cpu().item(), metrics - - return step_response - - def destroy(self): - self.training_engine.destroy() - self.inference_engine.destroy() - return - -# TODO: Once we are happy with this implementation, we can make this the new -# ArcticRLClient4VeRL. class ArcticRLClientWrapper: - """Thin wrapper around ArcticTraining's ArcticRLClient that exposes the - same interface as ArcticRLClient4VeRL so it can be used as a drop-in - replacement. - - Set USE_ARCTIC_TRAINING_CLIENT=1 env var to activate. + """Thin wrapper around ArcticTraining's ArcticRLClient """ def __init__(self, config): diff --git a/verl/workers/rollout/arctic_rollout/arctic_rollout.py b/verl/workers/rollout/arctic_rollout/arctic_rollout.py index c744b4a00fd..6abcf86302b 100644 --- a/verl/workers/rollout/arctic_rollout/arctic_rollout.py +++ b/verl/workers/rollout/arctic_rollout/arctic_rollout.py @@ -1,10 +1,10 @@ -import ray +import ray from typing import Any, Optional from verl.workers.rollout.vllm_rollout.vllm_async_server import vLLMHttpServer import argparse from typing import Any, Optional -from verl.trainer.ppo.arctic_rl_client import ArcticRLClient4VeRL +from verl.trainer.ppo.arctic_rl_client import ArcticRLClientWrapper from collections.abc import AsyncGenerator import ray @@ -31,7 +31,7 @@ class ArcticLLMEngine: def __init__( self, replica_rank: int, - arctic_rl_client: ArcticRLClient4VeRL, + arctic_rl_client: ArcticRLClientWrapper, ): self.replica_rank = replica_rank self.arctic_rl_client = arctic_rl_client @@ -88,7 +88,7 @@ def __init__( config: RolloutConfig, model_config: HFModelConfig, rollout_mode: RolloutMode, - arctic_rl_client: ArcticRLClient4VeRL, + arctic_rl_client: ArcticRLClientWrapper, workers: list[ActorHandle] = [], replica_rank: int = 0, node_rank: int = 0, @@ -148,24 +148,24 @@ def __init__( # f"data_parallel_rpc_port: {self._dp_rpc_port}, data_parallel_master_port: {self._dp_master_port}" # ) - def get_master_address(self): pass + def get_master_address(self): pass - def get_server_address(self): pass + def get_server_address(self): pass @property - def lora_as_adapter(self) -> bool: pass + def lora_as_adapter(self) -> bool: pass async def collective_rpc( self, **kwargs, ): - pass + pass async def launch_server(self, master_address: str = None, master_port: int = None, dp_rpc_port: int = None): - pass + pass async def run_server(self, args: argparse.Namespace): - pass + pass async def generate( @@ -319,4 +319,3 @@ async def sleep(self): async def abort_request(self, request_id: str) -> dict[str, Any]: return {"aborted": True, "request_id": 0} - \ No newline at end of file From 62bdb5b66c941bbf9729b9fb52b9ab1e090d1aad Mon Sep 17 00:00:00 2001 From: Tunji Ruwase Date: Tue, 21 Apr 2026 20:29:00 -0400 Subject: [PATCH 46/58] Disable CI (#10) --- .github/{workflows => workflows_old}/README.md | 0 .github/{workflows => workflows_old}/check-pr-title.yml | 0 .github/{workflows => workflows_old}/cpu_unit_tests.yml | 0 .github/{workflows => workflows_old}/doc.yml | 0 .github/{workflows => workflows_old}/docker-build-ascend-a2.yml | 0 .github/{workflows => workflows_old}/docker-build-ascend-a3.yml | 0 .github/{workflows => workflows_old}/e2e_ascend.yml | 0 .github/{workflows => workflows_old}/e2e_fully_async_policy.yml | 0 .github/{workflows => workflows_old}/e2e_one_step_off_policy.yml | 0 .../e2e_one_step_off_policy_ascend.yml | 0 .../{workflows => workflows_old}/e2e_ppo_grpo_trainer_trtllm.yml | 0 .github/{workflows => workflows_old}/e2e_ppo_trainer.yml | 0 .../e2e_ppo_trainer_megatron_sglang.yml | 0 .../e2e_ppo_trainer_megatron_sglang_2.yml | 0 .../e2e_ppo_trainer_megatron_vllm.yml | 0 .../e2e_ppo_trainer_megatron_vllm_2.yml | 0 .../e2e_ppo_trainer_megatron_vllm_2_ascend.yml | 0 .../{workflows => workflows_old}/e2e_ppo_trainer_veomni_vllm.yml | 0 .github/{workflows => workflows_old}/e2e_sft_llm.yml | 0 .github/{workflows => workflows_old}/e2e_sft_llm_ascend.yml | 0 .github/{workflows => workflows_old}/e2e_sft_vlm.yml | 0 .github/{workflows => workflows_old}/gpu_unit_tests.yml | 0 .github/{workflows => workflows_old}/model.yml | 0 .github/{workflows => workflows_old}/model_ascend.yml | 0 .github/{workflows => workflows_old}/nightly_ascend.yml | 0 .github/{workflows => workflows_old}/npu_unit_tests.yml | 0 .github/{workflows => workflows_old}/precommit-autofix.yml | 0 .github/{workflows => workflows_old}/reward_model_sglang.yml | 0 .github/{workflows => workflows_old}/reward_model_vllm.yml | 0 .github/{workflows => workflows_old}/reward_model_vllm_ascend.yml | 0 .github/{workflows => workflows_old}/sanity.yml | 0 .github/{workflows => workflows_old}/scorecard.yml | 0 .github/{workflows => workflows_old}/secrets_scan.yml | 0 .github/{workflows => workflows_old}/sgl.yml | 0 .github/{workflows => workflows_old}/type-coverage-check.yml | 0 .github/{workflows => workflows_old}/vllm.yml | 0 36 files changed, 0 insertions(+), 0 deletions(-) rename .github/{workflows => workflows_old}/README.md (100%) rename .github/{workflows => workflows_old}/check-pr-title.yml (100%) rename .github/{workflows => workflows_old}/cpu_unit_tests.yml (100%) rename .github/{workflows => workflows_old}/doc.yml (100%) rename .github/{workflows => workflows_old}/docker-build-ascend-a2.yml (100%) rename .github/{workflows => workflows_old}/docker-build-ascend-a3.yml (100%) rename .github/{workflows => workflows_old}/e2e_ascend.yml (100%) rename .github/{workflows => workflows_old}/e2e_fully_async_policy.yml (100%) rename .github/{workflows => workflows_old}/e2e_one_step_off_policy.yml (100%) rename .github/{workflows => workflows_old}/e2e_one_step_off_policy_ascend.yml (100%) rename .github/{workflows => workflows_old}/e2e_ppo_grpo_trainer_trtllm.yml (100%) rename .github/{workflows => workflows_old}/e2e_ppo_trainer.yml (100%) rename .github/{workflows => workflows_old}/e2e_ppo_trainer_megatron_sglang.yml (100%) rename .github/{workflows => workflows_old}/e2e_ppo_trainer_megatron_sglang_2.yml (100%) rename .github/{workflows => workflows_old}/e2e_ppo_trainer_megatron_vllm.yml (100%) rename .github/{workflows => workflows_old}/e2e_ppo_trainer_megatron_vllm_2.yml (100%) rename .github/{workflows => workflows_old}/e2e_ppo_trainer_megatron_vllm_2_ascend.yml (100%) rename .github/{workflows => workflows_old}/e2e_ppo_trainer_veomni_vllm.yml (100%) rename .github/{workflows => workflows_old}/e2e_sft_llm.yml (100%) rename .github/{workflows => workflows_old}/e2e_sft_llm_ascend.yml (100%) rename .github/{workflows => workflows_old}/e2e_sft_vlm.yml (100%) rename .github/{workflows => workflows_old}/gpu_unit_tests.yml (100%) rename .github/{workflows => workflows_old}/model.yml (100%) rename .github/{workflows => workflows_old}/model_ascend.yml (100%) rename .github/{workflows => workflows_old}/nightly_ascend.yml (100%) rename .github/{workflows => workflows_old}/npu_unit_tests.yml (100%) rename .github/{workflows => workflows_old}/precommit-autofix.yml (100%) rename .github/{workflows => workflows_old}/reward_model_sglang.yml (100%) rename .github/{workflows => workflows_old}/reward_model_vllm.yml (100%) rename .github/{workflows => workflows_old}/reward_model_vllm_ascend.yml (100%) rename .github/{workflows => workflows_old}/sanity.yml (100%) rename .github/{workflows => workflows_old}/scorecard.yml (100%) rename .github/{workflows => workflows_old}/secrets_scan.yml (100%) rename .github/{workflows => workflows_old}/sgl.yml (100%) rename .github/{workflows => workflows_old}/type-coverage-check.yml (100%) rename .github/{workflows => workflows_old}/vllm.yml (100%) diff --git a/.github/workflows/README.md b/.github/workflows_old/README.md similarity index 100% rename from .github/workflows/README.md rename to .github/workflows_old/README.md diff --git a/.github/workflows/check-pr-title.yml b/.github/workflows_old/check-pr-title.yml similarity index 100% rename from .github/workflows/check-pr-title.yml rename to .github/workflows_old/check-pr-title.yml diff --git a/.github/workflows/cpu_unit_tests.yml b/.github/workflows_old/cpu_unit_tests.yml similarity index 100% rename from .github/workflows/cpu_unit_tests.yml rename to .github/workflows_old/cpu_unit_tests.yml diff --git a/.github/workflows/doc.yml b/.github/workflows_old/doc.yml similarity index 100% rename from .github/workflows/doc.yml rename to .github/workflows_old/doc.yml diff --git a/.github/workflows/docker-build-ascend-a2.yml b/.github/workflows_old/docker-build-ascend-a2.yml similarity index 100% rename from .github/workflows/docker-build-ascend-a2.yml rename to .github/workflows_old/docker-build-ascend-a2.yml diff --git a/.github/workflows/docker-build-ascend-a3.yml b/.github/workflows_old/docker-build-ascend-a3.yml similarity index 100% rename from .github/workflows/docker-build-ascend-a3.yml rename to .github/workflows_old/docker-build-ascend-a3.yml diff --git a/.github/workflows/e2e_ascend.yml b/.github/workflows_old/e2e_ascend.yml similarity index 100% rename from .github/workflows/e2e_ascend.yml rename to .github/workflows_old/e2e_ascend.yml diff --git a/.github/workflows/e2e_fully_async_policy.yml b/.github/workflows_old/e2e_fully_async_policy.yml similarity index 100% rename from .github/workflows/e2e_fully_async_policy.yml rename to .github/workflows_old/e2e_fully_async_policy.yml diff --git a/.github/workflows/e2e_one_step_off_policy.yml b/.github/workflows_old/e2e_one_step_off_policy.yml similarity index 100% rename from .github/workflows/e2e_one_step_off_policy.yml rename to .github/workflows_old/e2e_one_step_off_policy.yml diff --git a/.github/workflows/e2e_one_step_off_policy_ascend.yml b/.github/workflows_old/e2e_one_step_off_policy_ascend.yml similarity index 100% rename from .github/workflows/e2e_one_step_off_policy_ascend.yml rename to .github/workflows_old/e2e_one_step_off_policy_ascend.yml diff --git a/.github/workflows/e2e_ppo_grpo_trainer_trtllm.yml b/.github/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml similarity index 100% rename from .github/workflows/e2e_ppo_grpo_trainer_trtllm.yml rename to .github/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml diff --git a/.github/workflows/e2e_ppo_trainer.yml b/.github/workflows_old/e2e_ppo_trainer.yml similarity index 100% rename from .github/workflows/e2e_ppo_trainer.yml rename to .github/workflows_old/e2e_ppo_trainer.yml diff --git a/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml b/.github/workflows_old/e2e_ppo_trainer_megatron_sglang.yml similarity index 100% rename from .github/workflows/e2e_ppo_trainer_megatron_sglang.yml rename to .github/workflows_old/e2e_ppo_trainer_megatron_sglang.yml diff --git a/.github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml b/.github/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml similarity index 100% rename from .github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml rename to .github/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml diff --git a/.github/workflows/e2e_ppo_trainer_megatron_vllm.yml b/.github/workflows_old/e2e_ppo_trainer_megatron_vllm.yml similarity index 100% rename from .github/workflows/e2e_ppo_trainer_megatron_vllm.yml rename to .github/workflows_old/e2e_ppo_trainer_megatron_vllm.yml diff --git a/.github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml b/.github/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml similarity index 100% rename from .github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml rename to .github/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml diff --git a/.github/workflows/e2e_ppo_trainer_megatron_vllm_2_ascend.yml b/.github/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml similarity index 100% rename from .github/workflows/e2e_ppo_trainer_megatron_vllm_2_ascend.yml rename to .github/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml diff --git a/.github/workflows/e2e_ppo_trainer_veomni_vllm.yml b/.github/workflows_old/e2e_ppo_trainer_veomni_vllm.yml similarity index 100% rename from .github/workflows/e2e_ppo_trainer_veomni_vllm.yml rename to .github/workflows_old/e2e_ppo_trainer_veomni_vllm.yml diff --git a/.github/workflows/e2e_sft_llm.yml b/.github/workflows_old/e2e_sft_llm.yml similarity index 100% rename from .github/workflows/e2e_sft_llm.yml rename to .github/workflows_old/e2e_sft_llm.yml diff --git a/.github/workflows/e2e_sft_llm_ascend.yml b/.github/workflows_old/e2e_sft_llm_ascend.yml similarity index 100% rename from .github/workflows/e2e_sft_llm_ascend.yml rename to .github/workflows_old/e2e_sft_llm_ascend.yml diff --git a/.github/workflows/e2e_sft_vlm.yml b/.github/workflows_old/e2e_sft_vlm.yml similarity index 100% rename from .github/workflows/e2e_sft_vlm.yml rename to .github/workflows_old/e2e_sft_vlm.yml diff --git a/.github/workflows/gpu_unit_tests.yml b/.github/workflows_old/gpu_unit_tests.yml similarity index 100% rename from .github/workflows/gpu_unit_tests.yml rename to .github/workflows_old/gpu_unit_tests.yml diff --git a/.github/workflows/model.yml b/.github/workflows_old/model.yml similarity index 100% rename from .github/workflows/model.yml rename to .github/workflows_old/model.yml diff --git a/.github/workflows/model_ascend.yml b/.github/workflows_old/model_ascend.yml similarity index 100% rename from .github/workflows/model_ascend.yml rename to .github/workflows_old/model_ascend.yml diff --git a/.github/workflows/nightly_ascend.yml b/.github/workflows_old/nightly_ascend.yml similarity index 100% rename from .github/workflows/nightly_ascend.yml rename to .github/workflows_old/nightly_ascend.yml diff --git a/.github/workflows/npu_unit_tests.yml b/.github/workflows_old/npu_unit_tests.yml similarity index 100% rename from .github/workflows/npu_unit_tests.yml rename to .github/workflows_old/npu_unit_tests.yml diff --git a/.github/workflows/precommit-autofix.yml b/.github/workflows_old/precommit-autofix.yml similarity index 100% rename from .github/workflows/precommit-autofix.yml rename to .github/workflows_old/precommit-autofix.yml diff --git a/.github/workflows/reward_model_sglang.yml b/.github/workflows_old/reward_model_sglang.yml similarity index 100% rename from .github/workflows/reward_model_sglang.yml rename to .github/workflows_old/reward_model_sglang.yml diff --git a/.github/workflows/reward_model_vllm.yml b/.github/workflows_old/reward_model_vllm.yml similarity index 100% rename from .github/workflows/reward_model_vllm.yml rename to .github/workflows_old/reward_model_vllm.yml diff --git a/.github/workflows/reward_model_vllm_ascend.yml b/.github/workflows_old/reward_model_vllm_ascend.yml similarity index 100% rename from .github/workflows/reward_model_vllm_ascend.yml rename to .github/workflows_old/reward_model_vllm_ascend.yml diff --git a/.github/workflows/sanity.yml b/.github/workflows_old/sanity.yml similarity index 100% rename from .github/workflows/sanity.yml rename to .github/workflows_old/sanity.yml diff --git a/.github/workflows/scorecard.yml b/.github/workflows_old/scorecard.yml similarity index 100% rename from .github/workflows/scorecard.yml rename to .github/workflows_old/scorecard.yml diff --git a/.github/workflows/secrets_scan.yml b/.github/workflows_old/secrets_scan.yml similarity index 100% rename from .github/workflows/secrets_scan.yml rename to .github/workflows_old/secrets_scan.yml diff --git a/.github/workflows/sgl.yml b/.github/workflows_old/sgl.yml similarity index 100% rename from .github/workflows/sgl.yml rename to .github/workflows_old/sgl.yml diff --git a/.github/workflows/type-coverage-check.yml b/.github/workflows_old/type-coverage-check.yml similarity index 100% rename from .github/workflows/type-coverage-check.yml rename to .github/workflows_old/type-coverage-check.yml diff --git a/.github/workflows/vllm.yml b/.github/workflows_old/vllm.yml similarity index 100% rename from .github/workflows/vllm.yml rename to .github/workflows_old/vllm.yml From 8043033d6adc65c7060d9f4a36f35c1b70ef5e3e Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 23 Apr 2026 15:44:54 +0000 Subject: [PATCH 47/58] Removing TrainingWorker --- .../arctic_rl/run_bird_grpo_arl_zorro_yes.sh | 8 +- verl/trainer/ppo/arctic_rl_client.py | 14 +- verl/trainer/ppo/arctic_trainer.py | 35 +- verl/workers/arctic_workers.py | 413 ++++++------------ 4 files changed, 165 insertions(+), 305 deletions(-) diff --git a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh index 0eabeb33692..8c17d88740d 100755 --- a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh +++ b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh @@ -34,7 +34,7 @@ COLOCATE=False # BSZ=32 # ROLL_N=16 BSZ=4 -ROLL_N=2 +ROLL_N=4 MAX_STEPS=10 PROMPT_LEN=16384 RESPONSE_LEN=4096 @@ -142,8 +142,8 @@ python3 -m verl.trainer.main_ppo \ custom_reward_function.name=compute_score \ trainer.total_training_steps=$MAX_STEPS \ arctic_rl.colocate=$COLOCATE \ - arctic_rl.training_gpus=1\ - arctic_rl.sampling_gpus=2\ - arctic_rl.log_prob_gpus=1\ + arctic_rl.training_gpus=2\ + arctic_rl.sampling_gpus=4\ + arctic_rl.log_prob_gpus=2\ arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ "$@" 2>&1 | tee $experiment_name.log diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index c74fd669266..a6914680b3d 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -3,7 +3,6 @@ import torch from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer from deepspeed.utils import OnDevice -from dss_client.client import DSSInferenceClient, DSSTrainingClient, DSSLogProbClient import ray from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy from ray.util.placement_group import placement_group @@ -20,12 +19,6 @@ def create_arctic_rl_client(config): ), )(ArcticRLClientWrapper).remote(config) -def create_meta_model(name_or_path: str): - model_config = AutoConfig.from_pretrained(name_or_path) - with OnDevice(dtype=torch.float16, device='meta'): - meta_model = AutoModelForCausalLM.from_config(model_config) - return meta_model - class ArcticRLClientWrapper: """Thin wrapper around ArcticTraining's ArcticRLClient @@ -206,9 +199,10 @@ def save_checkpoint(self): return response def update_weights(self): - response = self._client.sync_weights() - print(f"[ArcticRLClientWrapper] update_weights OUTPUT: {response.keys()=}") - return response + return None # TODO: Implement this + # response = self._client.sync_weights() + # print(f"[ArcticRLClientWrapper] update_weights OUTPUT: {response.keys()=}") + # return response def destroy(self): if self._client is not None: diff --git a/verl/trainer/ppo/arctic_trainer.py b/verl/trainer/ppo/arctic_trainer.py index f4e0cc0d187..970f30ec1bd 100644 --- a/verl/trainer/ppo/arctic_trainer.py +++ b/verl/trainer/ppo/arctic_trainer.py @@ -1,17 +1,10 @@ -import torch from typing import Optional from torch.utils.data import Dataset, Sampler from verl.trainer.ppo.ray_trainer import RayPPOTrainer -from verl.single_controller.ray import RayClassWithInitArgs, RayWorkerGroup, ResourcePoolManager -from verl.workers.arctic_workers import ActorRolloutRefWorker +from verl.single_controller.ray import RayWorkerGroup, ResourcePoolManager from verl.trainer.ppo.utils import Role, WorkerType -from omegaconf import OmegaConf -from verl.single_controller.ray.base import create_colocated_worker_cls from verl.trainer.ppo.arctic_rl_client import create_arctic_rl_client -def my_pdb(): - return - import pdb; pdb.set_trace() class ArcticPPOTrainer(RayPPOTrainer): def __init__( @@ -27,25 +20,25 @@ def __init__( collate_fn=None, train_sampler: Optional[Sampler] = None, device_name=None, - ): - super().__init__(config=config, - tokenizer=tokenizer, - processor=processor, - role_worker_mapping=role_worker_mapping, - resource_pool_manager=resource_pool_manager, - ray_worker_group_cls=ray_worker_group_cls, - train_dataset=train_dataset, - val_dataset=val_dataset, - collate_fn=collate_fn, - train_sampler=train_sampler, + ): + super().__init__(config=config, + tokenizer=tokenizer, + processor=processor, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + train_dataset=train_dataset, + val_dataset=val_dataset, + collate_fn=collate_fn, + train_sampler=train_sampler, device_name=device_name) self.use_gpu = False self.rl_client = create_arctic_rl_client(config=config) self.rl_client.initialize.remote(model_name="Qwen/Qwen3-0.6B") self.wg_kwargs["arctic_rl_client"] = self.rl_client - + def destroy(self): - # self.actor_rollout_wg.destroy() + # self.actor_rollout_wg.destroy() self.rl_client.destroy.remote() \ No newline at end of file diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index db2527bf83e..820ef6b5c21 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -71,24 +71,6 @@ def create_meta_model(name_or_path: str): return meta_model -DATA_PROTO_KEYS = ["gen_batch_output", "old_log_prob", "ref_log_prob", "compute_advantage", "actor_output"] -TENSOR_DICT_KEYS = ["full_log_prob", "full_ref_log_prob", "full_actor_output"] - -def load_dump_data(train_batch_size, roll_n) -> dict[str, DataProto]: - global_step = 1 - dump_data = {} - dump_path = os.path.join('/code/users/truwase/data/at_verl_dump', f'tbs{train_batch_size}_n{roll_n}') - dump_dir = Path(dump_path) - os.path.exists(dump_dir) - for key in DATA_PROTO_KEYS: - dump_data[key] = DataProto.load_from_disk(Path(dump_dir, f"{global_step}_{key}.pt")) - for key in TENSOR_DICT_KEYS: - dump_data[key] = torch.load(Path(dump_dir, f"{global_step}_{key}.pt"), weights_only=False) - - return dump_data - - - def no_padding_2_padding_prompt_response(tensor: torch.Tensor, data: TensorDict, pad_token_id) -> torch.Tensor: """Convert jagged tensor into a left padded prompt and right padded prompt of [bsz, max_response_len], which looks like tensor([ @@ -279,109 +261,162 @@ def prepare_padded_dss_batch_dict(data: TensorDict, pad_token_id) -> dict: return dss_batch_dict, max_prompt_len, max_response_len -class TrainingWorker(Worker, DistProfilerExtension): - """ - TrainingWorker provides a Tinker-like API (https://thinkingmachines.ai/tinker/) as a RayWorkerGroup - to a single controller. Currently, we only provide more coarse grained APIs, - and do not provide exact APIs as Tinker does. But this can be added in the future. - """ - def __init__(self, config: TrainingWorkerConfig, actor_config: ActorConfig, arctic_rl_client, tokenizer): +class ActorRolloutRefWorker(Worker, DistProfilerExtension): + def __init__(self, config: DictConfig, role: str, **kwargs): Worker.__init__(self) - - from verl.workers.engine import BaseEngine, EngineRegistry - - #initialize_global_process_group_ray(timeout_second=None) - self.config = config - self.actor_config = actor_config + self.role = role + self._is_actor = self.role in ["actor", "actor_rollout", "actor_rollout_ref"] + self._is_rollout = self.role in ["rollout", "actor_rollout", "actor_rollout_ref"] + self._is_ref = self.role in ["ref", "actor_rollout_ref"] - self.arctic_rl_client = arctic_rl_client - self.tokenizer = tokenizer - self.pad_token_id = self.tokenizer.pad_token_id + self.arctic_rl_client = kwargs.get("arctic_rl_client", None) + assert self.arctic_rl_client is not None, "arctic_rl_client is required" - self.model_config = self.config.model_config - self.engine_config = self.config.engine_config - self.optimizer_config = self.config.optimizer_config - self.checkpoint_config = self.config.checkpoint_config - self.device_name = get_device_name() self.use_zorro = ray.get(self.arctic_rl_client.is_zorro_enabled.remote()) - print(f"{self.engine_config=}") + DistProfilerExtension.__init__(self, DistProfiler(rank=self.rank, config=None, tool_config=None)) - if self.engine_config is None: - assert self.optimizer_config is None - if self.config.auto_select_engine_optim_fn is None: - raise ValueError( - "engine_config is not provided and auto_select_engine_optim_fn is not set. " - "Cannot determine engine backend." - ) - # Support automatically select engine backend given model config - self.engine_config, self.optimizer_config = self.config.auto_select_engine_optim_fn( - self.model_config, self.device_name + if self._is_actor: + model_config: HFModelConfig = omega_conf_to_dataclass(self.config.model) + actor_config: ActorConfig = omega_conf_to_dataclass(self.config.actor) + actor_config.model_config = model_config + actor_training_config = TrainingWorkerConfig( + model_type="language_model", + model_config=actor_config.model_config, + engine_config=actor_config.engine, + optimizer_config=actor_config.optim, + checkpoint_config=actor_config.checkpoint, ) + self.actor_config = actor_config - # we use the one defined in model - # TODO: this is not elegant and should refactor later - self.engine_config.use_remove_padding = self.model_config.use_remove_padding - self.engine_config.use_fused_kernels = self.model_config.use_fused_kernels - - if repatch is not None: - # NPU MindSpeed patch, will be refactored with MindSpeedEngine. - repatch(self.engine_config.get("override_transformer_config", {})) + assert self.config.actor.use_dynamic_bsz == self.config.rollout.log_prob_use_dynamic_bsz - # TODO: add DistProfilerExtension - self.profiler_config = self.config.profiler_config - if self.profiler_config is not None: - self.profiler_tool_config = self.profiler_config.tool_config.get(self.profiler_config.tool, {}) - else: - self.profiler_tool_config = None + # assign engine configs + actor_training_config.engine_config.use_dynamic_bsz = self.config.actor.use_dynamic_bsz + actor_training_config.engine_config.infer_max_token_len_per_gpu = ( + self.config.rollout.log_prob_max_token_len_per_gpu + ) + actor_training_config.engine_config.infer_micro_batch_size_per_gpu = ( + self.config.rollout.log_prob_micro_batch_size_per_gpu + ) + actor_training_config.engine_config.max_token_len_per_gpu = self.config.actor.ppo_max_token_len_per_gpu + actor_training_config.engine_config.micro_batch_size_per_gpu = ( + self.config.actor.ppo_micro_batch_size_per_gpu + ) + actor_training_config.engine_config.use_remove_padding = model_config.use_remove_padding - DistProfilerExtension.__init__( - self, DistProfiler(rank=self.rank, config=self.profiler_config, tool_config=self.profiler_tool_config) - ) + if self.config.actor.use_dynamic_bsz: + assert self.config.rollout.log_prob_max_token_len_per_gpu is not None + assert self.config.actor.ppo_max_token_len_per_gpu is not None + else: + assert self.config.rollout.log_prob_micro_batch_size_per_gpu is not None + assert self.config.actor.ppo_micro_batch_size_per_gpu is not None - # self.engine: BaseEngine = EngineRegistry.new( - # model_type=self.config.model_type, - # backend=self.engine_config.strategy, - # model_config=self.model_config, - # engine_config=self.engine_config, - # optimizer_config=self.optimizer_config, - # checkpoint_config=self.checkpoint_config, - # ) + trust_remote_code=self.config.model.get("trust_remote_code", False) + self.tokenizer = hf_tokenizer(self.config.model.path, trust_remote_code=trust_remote_code) + if self.tokenizer.pad_token_id is None: + self.tokenizer.pad_token_id = self.tokenizer.eos_token_id + self.pad_token_id = self.tokenizer.pad_token_id - # # build dispatch info - # self._register_dispatch_collect_info( - # mesh_name="train", - # dp_rank=self.engine.get_data_parallel_rank(), - # is_collect=self.engine.is_mp_src_rank_with_outputs(), - # ) + self.device_name = get_device_name() + self.flops_counter = FlopsCounter(model_config.hf_config) - self.flops_counter = FlopsCounter(self.model_config.hf_config) - self.loss_fn = None @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def to(self, device, model=True, optimizer=True, grad=True): - """Manual control of load/offload""" - assert device in ["cpu", "device"] + def init_model(self): + self._register_dispatch_collect_info("actor", dp_rank=self.rank, is_collect=True) + self._register_dispatch_collect_info("ref", dp_rank=self.rank, is_collect=True) + self._register_dispatch_collect_info("rollout", dp_rank=self.rank, is_collect=True) - if device == "device": - device = get_device_name() + return - self.engine.to(device=device, model=model, optimizer=optimizer, grad=grad) + @register(dispatch_mode=Dispatch.ONE_TO_ALL) + def destroy(self): + self.dss_training_engine.destroy() + self.arctic_inference_engine.destroy() + return @register(dispatch_mode=Dispatch.ONE_TO_ALL) def set_loss_fn(self, loss_fn): - self.loss_fn = loss_fn + return @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def reset(self): - """ - Reset the model engine to the initial state. If the engine is not initialized, - we initialize it. Otherwise, reload ckpt and reset states - """ - pass # self.engine.initialize() + def to(self, device, model=True, optimizer=True, grad=True): + """Manual control of load/offload""" + return + + + def _update_config_params(self, data: TensorDict): + default_keys = dict( + use_remove_padding=self.config.model.use_remove_padding, + use_dynamic_bsz=self.config.actor.use_dynamic_bsz, + max_token_len_per_gpu=self.config.actor.ppo_max_token_len_per_gpu, + micro_batch_size_per_gpu=self.config.actor.ppo_micro_batch_size_per_gpu, + use_fused_kernels=self.config.actor.use_fused_kernels, + ) + + for key, val in default_keys.items(): + if key not in data.keys(): + tu.assign_non_tensor(data, **{key: val}) + + + def compute_any_log_prob(self, data: TensorDict, compute_log_prob_fn) -> TensorDict: + # print(f"compute_ref_log_prob data: {data}") + batch, max_prompt_len, max_response_len = prepare_padded_dss_batch_dict(data, self.pad_token_id) + + self._update_config_params(data) + + #max_token_len_per_gpu = self.actor_config.ppo_max_token_len_per_gpu + + meta = dict( + rollout_n=self.config.rollout.n, + max_prompt_len=max_prompt_len, + max_response_len=max_response_len, + max_token_len_per_gpu=data["max_token_len_per_gpu"], + temperature=data["temperature"], + ) + + payload = dict(batch=batch, meta=meta) + + response = ray.get(compute_log_prob_fn.remote(payload)) + + # print(f"compute_any_log_prob: {response['batch']['entropy'].shape=} {response['batch']['log_probs'].shape=}") + + #batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) + #model_output = batch_output.pop("model_output", {}) + + # verl wants a full [bs, max_prompt_len+max_response_len] tensors and jagged + entropy = prepand_max_prompt_len_zeros(response['batch']['entropy'], max_prompt_len) + log_probs = prepand_max_prompt_len_zeros(response['batch']['log_probs'], max_prompt_len) + # print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") + entropy = make_njt(data, entropy) + log_probs = make_njt(data, log_probs) + # print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") + + model_output = dict(entropy=entropy, log_probs=log_probs) + metrics = response['metrics'] + # TODO: fix me - mfu is not computed here + metrics["mfu"] = 0.0 + + final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) + + return final_output + + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="ref")) + @DistProfiler.annotate(color="olive", role="ref_compute_log_prob") + def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: + return self.compute_any_log_prob(data, self.arctic_rl_client.compute_ref_log_prob) + + + @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) + @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") + def compute_log_prob(self, data: TensorDict) -> TensorDict: + return self.compute_any_log_prob(data, self.arctic_rl_client.compute_log_prob) + def _postprocess_output(self, output, *, global_token_num, delta_time, forward_only, images_seqlens): """ @@ -436,7 +471,7 @@ def _postprocess_output(self, output, *, global_token_num, delta_time, forward_o @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="train"), blocking=False) - def train_global_batch(self, data: TensorDict) -> TensorDict: + def train_actor_global_batch(self, data: TensorDict) -> TensorDict: """Train a global batch Args: @@ -445,12 +480,8 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: Returns: """ - assert self.loss_fn is not None, "loss function can't be None when calling train_global_batch" - disable_auto_offload = tu.pop(data, key="disable_auto_offload", default=False) - self.engine_config = self.config.engine_config - # update global_token_num = data["input_ids"].offsets().diff().tolist() # (total_nnz,) tu.assign_non_tensor( @@ -467,11 +498,11 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: # inject engineering parameters if not specified default_keys = dict( - use_remove_padding=self.model_config.use_remove_padding, - use_dynamic_bsz=self.engine_config.use_dynamic_bsz, - max_token_len_per_gpu=self.engine_config.max_token_len_per_gpu, - micro_batch_size_per_gpu=self.engine_config.micro_batch_size_per_gpu, - use_fused_kernels=self.engine_config.use_fused_kernels, + use_remove_padding=self.config.model.use_remove_padding, + use_dynamic_bsz=self.config.actor.use_dynamic_bsz, + max_token_len_per_gpu=self.config.actor.ppo_max_token_len_per_gpu, + micro_batch_size_per_gpu=self.config.actor.ppo_micro_batch_size_per_gpu, + use_fused_kernels=self.config.actor.use_fused_kernels, ) for key, val in default_keys.items(): @@ -531,14 +562,14 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: old_log_probs=data["old_log_probs"], advantages=data["advantages"], ) - if self.actor_config.use_kl_loss: + if self.config.actor.use_kl_loss: batch["ref_log_prob"] = data["ref_log_prob"] # print(f"{batch=}") # TODO: move to init since globally constant meta = dict( - rollout_n=self.actor_config.rollout_n, + rollout_n=self.config.rollout.n, max_prompt_len=max_prompt_len, max_response_len=max_response_len, max_token_len_per_gpu=data["max_token_len_per_gpu"], @@ -551,7 +582,7 @@ def train_global_batch(self, data: TensorDict) -> TensorDict: # we need to serialize the config object to dict # dataclasses.asdict only returns keys that are defined at init (vars will do more) - but perhaps we want `asdict`? - actor_config_as_dict = vars(self.actor_config) + actor_config_as_dict = vars(self.config.actor) # print(f"update_actor: {self.actor_config=}") # print(f"update_actor: {actor_config_as_dict=}") import json @@ -559,7 +590,7 @@ def safe_serialize(obj): return json.loads(json.dumps(obj, default=lambda o: None)) actor_config_as_dict = safe_serialize(actor_config_as_dict) - policy_loss_config = safe_serialize(vars(self.actor_config.policy_loss)) + policy_loss_config = safe_serialize(vars(self.config.actor.policy_loss)) meta.update(dict(actor_config=actor_config_as_dict, policy_loss_config=policy_loss_config)) # print(f"update_actor: {post_process_inputs=}") @@ -648,169 +679,11 @@ def safe_serialize(obj): return output - -class ActorRolloutRefWorker(Worker, DistProfilerExtension): - def __init__(self, config: DictConfig, role: str, **kwargs): - Worker.__init__(self) - self.config = config - self.role = role - self._is_actor = self.role in ["actor", "actor_rollout", "actor_rollout_ref"] - self._is_rollout = self.role in ["rollout", "actor_rollout", "actor_rollout_ref"] - self._is_ref = self.role in ["ref", "actor_rollout_ref"] - - self.arctic_rl_client = kwargs.get("arctic_rl_client", None) - - # assert self.arctic_rl_client is not None, "arctic_rl_client is required" - self._loaded_dump_data = load_dump_data(1, 1) - DistProfilerExtension.__init__(self, DistProfiler(rank=self.rank, config=None, tool_config=None)) - - if self._is_actor: - model_config: HFModelConfig = omega_conf_to_dataclass(self.config.model) - actor_config: ActorConfig = omega_conf_to_dataclass(self.config.actor) - actor_config.model_config = model_config - actor_training_config = TrainingWorkerConfig( - model_type="language_model", - model_config=actor_config.model_config, - engine_config=actor_config.engine, - optimizer_config=actor_config.optim, - checkpoint_config=actor_config.checkpoint, - ) - self.actor_config = actor_config - - assert self.config.actor.use_dynamic_bsz == self.config.rollout.log_prob_use_dynamic_bsz - - # assign engine configs - actor_training_config.engine_config.use_dynamic_bsz = self.config.actor.use_dynamic_bsz - actor_training_config.engine_config.infer_max_token_len_per_gpu = ( - self.config.rollout.log_prob_max_token_len_per_gpu - ) - actor_training_config.engine_config.infer_micro_batch_size_per_gpu = ( - self.config.rollout.log_prob_micro_batch_size_per_gpu - ) - actor_training_config.engine_config.max_token_len_per_gpu = self.config.actor.ppo_max_token_len_per_gpu - actor_training_config.engine_config.micro_batch_size_per_gpu = ( - self.config.actor.ppo_micro_batch_size_per_gpu - ) - actor_training_config.engine_config.use_remove_padding = model_config.use_remove_padding - - if self.config.actor.use_dynamic_bsz: - assert self.config.rollout.log_prob_max_token_len_per_gpu is not None - assert self.config.actor.ppo_max_token_len_per_gpu is not None - else: - assert self.config.rollout.log_prob_micro_batch_size_per_gpu is not None - assert self.config.actor.ppo_micro_batch_size_per_gpu is not None - - trust_remote_code=self.config.model.get("trust_remote_code", False) - self.tokenizer = hf_tokenizer(self.config.model.path, trust_remote_code=trust_remote_code) - if self.tokenizer.pad_token_id is None: - self.tokenizer.pad_token_id = self.tokenizer.eos_token_id - self.pad_token_id = self.tokenizer.pad_token_id - self.actor = TrainingWorker(config=actor_training_config, actor_config=actor_config, arctic_rl_client=self.arctic_rl_client, tokenizer=self.tokenizer ) - - self.actor.reset() - self.loss_fn = partial(ppo_loss, config=actor_config) - self.actor.set_loss_fn(loss_fn=self.loss_fn) - - self.set_dispatch_collect(mesh_name="actor", **self.actor.get_dispatch_collect()) - - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - self._register_dispatch_collect_info("actor", dp_rank=self.rank, is_collect=True) - self._register_dispatch_collect_info("ref", dp_rank=self.rank, is_collect=True) - self._register_dispatch_collect_info("rollout", dp_rank=self.rank, is_collect=True) - - return - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def destroy(self): - self.dss_training_engine.destroy() - self.arctic_inference_engine.destroy() - return - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def set_loss_fn(self, loss_fn): - return - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def to(self, device, model=True, optimizer=True, grad=True): - """Manual control of load/offload""" - return - - - def _update_config_params(self, data: TensorDict): - default_keys = dict( - use_remove_padding=self.actor.model_config.use_remove_padding, - use_dynamic_bsz=self.actor.engine_config.use_dynamic_bsz, - max_token_len_per_gpu=self.actor.engine_config.max_token_len_per_gpu, - micro_batch_size_per_gpu=self.actor.engine_config.micro_batch_size_per_gpu, - use_fused_kernels=self.actor.engine_config.use_fused_kernels, - ) - - for key, val in default_keys.items(): - if key not in data.keys(): - tu.assign_non_tensor(data, **{key: val}) - - - def compute_any_log_prob(self, data: TensorDict, compute_log_prob_fn) -> TensorDict: - # print(f"compute_ref_log_prob data: {data}") - batch, max_prompt_len, max_response_len = prepare_padded_dss_batch_dict(data, self.pad_token_id) - - self._update_config_params(data) - - #max_token_len_per_gpu = self.actor_config.ppo_max_token_len_per_gpu - - meta = dict( - rollout_n=self.actor_config.rollout_n, - max_prompt_len=max_prompt_len, - max_response_len=max_response_len, - max_token_len_per_gpu=data["max_token_len_per_gpu"], - temperature=data["temperature"], - ) - - payload = dict(batch=batch, meta=meta) - - response = ray.get(compute_log_prob_fn.remote(payload)) - - # print(f"compute_any_log_prob: {response['batch']['entropy'].shape=} {response['batch']['log_probs'].shape=}") - - #batch_output = postprocess_log_prob_output(data=data, entropy=entropy, log_probs=log_probs) - #model_output = batch_output.pop("model_output", {}) - - # verl wants a full [bs, max_prompt_len+max_response_len] tensors and jagged - entropy = prepand_max_prompt_len_zeros(response['batch']['entropy'], max_prompt_len) - log_probs = prepand_max_prompt_len_zeros(response['batch']['log_probs'], max_prompt_len) - # print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") - entropy = make_njt(data, entropy) - log_probs = make_njt(data, log_probs) - # print(f"compute_any_log_prob: {entropy.shape=} {log_probs.shape=}") - - model_output = dict(entropy=entropy, log_probs=log_probs) - metrics = response['metrics'] - # TODO: fix me - mfu is not computed here - metrics["mfu"] = 0.0 - - final_output = tu.get_tensordict(tensor_dict=model_output, non_tensor_dict={"metrics": metrics}) - - return final_output - - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="ref")) - @DistProfiler.annotate(color="olive", role="ref_compute_log_prob") - def compute_ref_log_prob(self, data: TensorDict) -> TensorDict: - return self.compute_any_log_prob(data, self.arctic_rl_client.compute_ref_log_prob) - - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) - @DistProfiler.annotate(color="blue", role="actor_compute_log_prob") - def compute_log_prob(self, data: TensorDict) -> TensorDict: - return self.compute_any_log_prob(data, self.arctic_rl_client.compute_log_prob) - - @register(dispatch_mode=make_nd_compute_dataproto_dispatch_fn(mesh_name="actor")) @DistProfiler.annotate(color="red", role="actor_update") def update_actor(self, data: TensorDict) -> TensorDict: - output = self.actor.train_global_batch(data=data) + # output = self.actor.train_global_batch(data=data) + output = self.train_actor_global_batch(data=data) return output.cpu() if output is not None else None # TODO: Load Checkpoint API From 0e60e1082ca74cb2f7e11b679348e08353eeeef2 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 23 Apr 2026 15:47:04 +0000 Subject: [PATCH 48/58] Cleanup --- verl/workers/arctic_workers.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/verl/workers/arctic_workers.py b/verl/workers/arctic_workers.py index 820ef6b5c21..8100518ef3d 100644 --- a/verl/workers/arctic_workers.py +++ b/verl/workers/arctic_workers.py @@ -1,14 +1,10 @@ from pathlib import Path import torch -from verl.utils.ray_utils import auto_await from verl.single_controller.base.decorator import Dispatch, make_nd_compute_dataproto_dispatch_fn, register -from verl.protocol import DataProto from verl.single_controller.base import Worker from verl.utils.profiler import DistProfiler, DistProfilerExtension -from verl.workers.engine_workers import ActorRolloutRefWorker as EngineActorRolloutRefWorker from omegaconf import DictConfig from tensordict import TensorDict -from dss_client.client import DSSInferenceClient, DSSTrainingClient from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer from deepspeed.utils import OnDevice from verl.utils import tensordict_utils as tu @@ -24,8 +20,6 @@ set_expandable_segments, ) from codetiming import Timer -import functools -import logging import os from contextlib import nullcontext from functools import partial From ca9c59deb282bd46f9b9805bd715f752550ab169 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 23 Apr 2026 15:52:40 +0000 Subject: [PATCH 49/58] Cleanup --- scripts/arctic_rl/install.sh | 5 ----- scripts/arctic_rl/setup_repos.sh | 4 ---- verl/workers/utils/padding.py | 12 ------------ 3 files changed, 21 deletions(-) delete mode 100644 scripts/arctic_rl/install.sh delete mode 100644 scripts/arctic_rl/setup_repos.sh diff --git a/scripts/arctic_rl/install.sh b/scripts/arctic_rl/install.sh deleted file mode 100644 index 1294c76819a..00000000000 --- a/scripts/arctic_rl/install.sh +++ /dev/null @@ -1,5 +0,0 @@ -uv pip install -e "./ArcticInference-internal[server]" -uv pip install -e ./dss-client -uv pip install -e ./ArcticTraining-dss -cd arctic-verl -/code/shared/verl_snowrlhf/install-h200.sh diff --git a/scripts/arctic_rl/setup_repos.sh b/scripts/arctic_rl/setup_repos.sh deleted file mode 100644 index 9d2c905bfd0..00000000000 --- a/scripts/arctic_rl/setup_repos.sh +++ /dev/null @@ -1,4 +0,0 @@ -git clone -b verl_integration https://github.com/snowflake-eng/dss-client.git -git clone -b tunji/verl_integration https://github.com/snowflake-eng/ArcticTraining-dss.git -git clone -b public https://github.com/snowflake-eng/ArcticInference-internal.git -git clone -b tunji/arl_client https://github.com/snowflake-eng/arctic-verl.git diff --git a/verl/workers/utils/padding.py b/verl/workers/utils/padding.py index d8e53e40c7b..4b3fa7b2f07 100644 --- a/verl/workers/utils/padding.py +++ b/verl/workers/utils/padding.py @@ -99,10 +99,6 @@ def no_padding_2_padding(tensor: torch.Tensor, data: TensorDict) -> torch.Tensor prompt_ids = data["prompts"] response_ids = data["responses"] attention_mask = data["attention_mask"] - # print(f"{prompt_ids.shape=}") - # print(f"{response_ids.shape=}") - # print(f"{attention_mask.shape=}") - # print(f"{attention_mask=}") max_response_len = tu.get_non_tensor_data(data=data, key="max_response_len", default=-1) @@ -120,14 +116,6 @@ def no_padding_2_padding(tensor: torch.Tensor, data: TensorDict) -> torch.Tensor sequence_lens = prompt_lens + response_lens sequence_offsets = sequence_lens.cumsum(dim=0) - # print(f"{data=}") - # print(f"{prompt_lens=}") - # print(f"{response_lens=}") - # print(f"{response_lens=}") - # print(f"{max_response_len=}") - # print(f"{sequence_offsets=}") - # print(f"{values=}") - # print(f"{values.shape=}") assert sequence_offsets[-1].item() == values.shape[0], f"{sequence_offsets[-1].item()} != {values.shape[0]}" response_list = [] From 355efd761faea7fdbacd8bcadb4179681c83292c Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 23 Apr 2026 15:53:19 +0000 Subject: [PATCH 50/58] Cleanup --- verl/workers/utils/padding.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/verl/workers/utils/padding.py b/verl/workers/utils/padding.py index 4b3fa7b2f07..f6883fbc748 100644 --- a/verl/workers/utils/padding.py +++ b/verl/workers/utils/padding.py @@ -102,7 +102,6 @@ def no_padding_2_padding(tensor: torch.Tensor, data: TensorDict) -> torch.Tensor max_response_len = tu.get_non_tensor_data(data=data, key="max_response_len", default=-1) - # print(f"{prompt_ids.is_nested=}") if prompt_ids.is_nested: prompt_lens = prompt_ids.offsets().diff() response_lens = response_ids.offsets().diff() @@ -116,7 +115,7 @@ def no_padding_2_padding(tensor: torch.Tensor, data: TensorDict) -> torch.Tensor sequence_lens = prompt_lens + response_lens sequence_offsets = sequence_lens.cumsum(dim=0) - assert sequence_offsets[-1].item() == values.shape[0], f"{sequence_offsets[-1].item()} != {values.shape[0]}" + assert sequence_offsets[-1].item() == values.shape[0] response_list = [] for resp_len, seq_offset in zip(response_lens, sequence_offsets, strict=True): From b930595229cadbc7a0e245ba421582283addb14d Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 23 Apr 2026 15:54:34 +0000 Subject: [PATCH 51/58] Cleanup --- verl/workers/engine_workers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verl/workers/engine_workers.py b/verl/workers/engine_workers.py index 5367479a5bf..d0c065e4dfd 100644 --- a/verl/workers/engine_workers.py +++ b/verl/workers/engine_workers.py @@ -667,7 +667,7 @@ async def update_weights(self, global_steps: int = None): log_gpu_memory_usage("After update_weights", logger=logger) # 3. offload model to cpu - self.actor.engine.to("cpu", model=self.actor.engine.is_param_offload_enabled, optimizer=False, grad=False) + self.actor.engine.to("cpu", model=True, optimizer=False, grad=False) aggressive_empty_cache(force_sync=True) # 4. resume kv_cache From 4a31ea0fd1a2e40433dcb56e7b14f822cc706f32 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Thu, 23 Apr 2026 16:00:45 +0000 Subject: [PATCH 52/58] Cleanup --- examples/arctic_rl/bird_reward.py | 272 ------------------ examples/arctic_rl/debug_at_gsm8k_grpo.sh | 101 ------- .../run_arctic_gsm8k_grpo_zorro_no.sh | 124 -------- .../run_arctic_gsm8k_grpo_zorro_yes.sh | 123 -------- ...un_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh | 138 --------- ...n_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh | 135 --------- .../arctic_rl/run_bird_grpo_arl_zorro_yes.sh | 149 ---------- examples/arctic_rl/run_bird_grpo_baseline.sh | 134 --------- examples/arctic_rl/run_gsm8k_grpo.sh | 103 ------- .../run_qwen3_1.7b_bird_grpo_baseline.sh | 132 --------- verl/workers/utils/padding.py | 1 - 11 files changed, 1412 deletions(-) delete mode 100644 examples/arctic_rl/bird_reward.py delete mode 100755 examples/arctic_rl/debug_at_gsm8k_grpo.sh delete mode 100755 examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh delete mode 100755 examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh delete mode 100755 examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh delete mode 100755 examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh delete mode 100755 examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh delete mode 100755 examples/arctic_rl/run_bird_grpo_baseline.sh delete mode 100755 examples/arctic_rl/run_gsm8k_grpo.sh delete mode 100755 examples/arctic_rl/run_qwen3_1.7b_bird_grpo_baseline.sh diff --git a/examples/arctic_rl/bird_reward.py b/examples/arctic_rl/bird_reward.py deleted file mode 100644 index 92a0bec4a30..00000000000 --- a/examples/arctic_rl/bird_reward.py +++ /dev/null @@ -1,272 +0,0 @@ -""" -SQL reward function for BIRD RL training, adapted from SnowflakeDialectSQLRewardManagerV6b. - -Uses SQLite execution instead of Snowflake. Compatible with verl's -custom_reward_function mechanism via compute_score(). - -Reward scheme (matching V6b non-semantic-model behavior): - 1.0 - Predicted SQL produces the same result set as gold SQL - 0.1 - Predicted SQL executes successfully but produces wrong results, - OR SQL was extracted successfully (format bonus) - 0.0 - No SQL extracted, SQL fails to execute, or timeout -""" - -import json -import re -import sqlite3 -from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError -from functools import lru_cache - -SQL_TIMEOUT = 30 -DEFAULT_LIMIT_NUMBER = 5000 -FORMAT_REWARD_BONUS = 0.1 - - -# --------------------------------------------------------------------------- -# SQL extraction (mirrors V6b's extract_solution / _extract_sql_omnisql) -# --------------------------------------------------------------------------- - -def _extract_sql_omnisql(message: str) -> str: - """Extract SQL from ```sql ... ``` markdown blocks (last valid block).""" - pattern = r"```sql\s*(.*?)\s*```" - sql_blocks = re.findall(pattern, message, re.DOTALL) - for block in reversed(sql_blocks): - if len(block.strip()) > 6: - return block.strip() - return "" - - -def _extract_sql_generic_block(message: str) -> str: - """Extract SQL from generic ``` ... ``` blocks containing SELECT.""" - blocks = re.findall(r"```\s*(.*?)\s*```", message, re.DOTALL) - for block in reversed(blocks): - if "SELECT" in block.upper() and len(block.strip()) > 6: - return block.strip() - return "" - - -def _extract_sql_analyst(message: str) -> str: - """Extract SQL from ```json { "sql": "..." } ``` blocks.""" - block = re.search(r"```\s*json(.*?)```", message, re.DOTALL) - if block is None: - return "" - json_str = block.group(1) - idx_left = json_str.rfind("{") - idx_right = json_str.find("}") - if idx_left == -1 or idx_right == -1: - return "" - json_str = json_str[idx_left : idx_right + 1] - try: - return json.loads(json_str.replace("\\n", "\n").replace("\\'", "'"), strict=False).get("sql", "") - except Exception: - return "" - - -def _extract_sql_raw_select(message: str) -> str: - """Fallback: extract a raw SELECT statement.""" - match = re.search(r"(SELECT\s+.+?)(?:\n\n|$)", message, re.DOTALL | re.IGNORECASE) - if match: - return match.group(1).strip() - return "" - - -def extract_sql(response: str) -> str: - """Extract SQL from model response, following V6b's extraction pipeline. - - 1. Split on to isolate the answer portion - 2. Try ```sql blocks - 3. Try generic ``` blocks with SELECT - 4. Try ```json blocks with {"sql": ...} - 5. Fallback to raw SELECT statement - """ - if "" in response: - answer_part = response.split("", 1)[1] - else: - answer_part = response - - sql = _extract_sql_omnisql(answer_part) - if sql: - return sql - - sql = _extract_sql_generic_block(answer_part) - if sql: - return sql - - sql = _extract_sql_analyst(answer_part) - if sql: - return sql - - return _extract_sql_raw_select(answer_part) - - -# --------------------------------------------------------------------------- -# Format validation (mirrors V6b's validate_response_structure) -# --------------------------------------------------------------------------- - -def validate_response_format(response: str) -> bool: - """Check that the response has exactly one ... pair, properly nested.""" - start_positions = [m.start() for m in re.finditer(r"", response)] - end_positions = [m.start() for m in re.finditer(r"", response)] - - if len(start_positions) != 1 or len(end_positions) != 1: - return False - return start_positions[0] < end_positions[0] - - -# --------------------------------------------------------------------------- -# LIMIT addition (mirrors V6b's _add_limit_to_query) -# --------------------------------------------------------------------------- - -def _add_limit_to_query(query: str, limit: int = DEFAULT_LIMIT_NUMBER) -> str: - """Add LIMIT clause if the query doesn't already have one.""" - if not query: - return query - upper = query.upper() - if "LIMIT " in upper or "LIMIT\n" in upper or "LIMIT\t" in upper: - return query - return query.rstrip().rstrip(";").rstrip() + f" LIMIT {limit};" - - -# --------------------------------------------------------------------------- -# SQLite execution and comparison -# --------------------------------------------------------------------------- - -def _execute_sql(db_path: str, sql: str, timeout: float = SQL_TIMEOUT) -> frozenset | Exception: - """Execute SQL against a SQLite database and return result as frozenset of row tuples.""" - try: - conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5) - deadline = __import__("time").monotonic() + timeout - def _check_cancel(): - if __import__("time").monotonic() > deadline: - return 1 - return 0 - conn.set_progress_handler(_check_cancel, 1000) - cursor = conn.cursor() - cursor.execute(sql) - rows = cursor.fetchall() - conn.close() - return frozenset(rows) - except sqlite3.OperationalError as e: - if "interrupt" in str(e).lower(): - return TimeoutError(f"SQL execution exceeded {timeout}s") - return e - except Exception as e: - return e - - -def _execute_with_timeout(db_path: str, sql: str, timeout: float = SQL_TIMEOUT) -> frozenset | Exception: - """Execute SQL with a timeout using a thread pool. - - Uses shutdown(wait=False) to avoid blocking if the SQLite thread is stuck. - The SQLite progress handler provides cooperative cancellation. - """ - executor = ThreadPoolExecutor(max_workers=1) - future = executor.submit(_execute_sql, db_path, sql, timeout) - try: - return future.result(timeout=timeout + 2) - except FuturesTimeoutError: - return TimeoutError(f"SQL execution exceeded {timeout}s") - except Exception as e: - return e - finally: - executor.shutdown(wait=False) - - -def _compare_results( - db_path: str, - pred_sql: str, - gold_sqls: list[str], - timeout: float = SQL_TIMEOUT, -) -> tuple[float, bool]: - """Execute and compare predicted SQL against all gold SQLs. - - Returns (reward, execution_success) matching V6b's non-semantic-model logic: - - 1.0 if pred result == any gold result (frozenset match) - - 0.1 if pred executes but doesn't match any gold - - 0.0 if pred fails to execute - - Caches gold results within this call to avoid re-execution. - """ - pred_sql_limited = _add_limit_to_query(pred_sql) - pred_result = _execute_with_timeout(db_path, pred_sql_limited, timeout) - - if isinstance(pred_result, Exception): - return 0.0, False - - gold_cache: dict[str, frozenset | Exception] = {} - scores = [] - for gold_sql in gold_sqls: - if gold_sql not in gold_cache: - gold_sql_limited = _add_limit_to_query(gold_sql) - gold_cache[gold_sql] = _execute_with_timeout(db_path, gold_sql_limited, timeout) - gold_result = gold_cache[gold_sql] - - if isinstance(gold_result, Exception): - scores.append(0.0) - continue - - if pred_result == gold_result: - scores.append(1.0) - else: - scores.append(0.1) - - return (max(scores) if scores else 0.0), True - - -# --------------------------------------------------------------------------- -# Main entry point -# --------------------------------------------------------------------------- - -def compute_score(data_source, solution_str, ground_truth, extra_info=None, **kwargs): - """Compute reward score for SQL generation (verl custom_reward_function interface). - - Mirrors SnowflakeDialectSQLRewardManagerV6b logic with SQLite execution: - 1. Extract SQL from model response - 2. Validate response format ( tags) - 3. Execute predicted and gold SQL against SQLite - 4. Compare results (frozenset match) - 5. Apply format bonus - - Args: - data_source: Dataset identifier (e.g. "bird") - solution_str: Full model response text (decoded) - ground_truth: Gold SQL query string - extra_info: Dict with at minimum {"db_path": "/path/to/db.sqlite"}. - Optionally {"alternative_answers": [...]} for multiple gold SQLs. - - Returns: - dict with "score" (float), "format_correct" (float), "execution_success" (float) - """ - if extra_info is None: - return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} - - db_path = extra_info.get("db_path", "") - if not db_path: - return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} - - pred_sql = extract_sql(solution_str) - format_correct = float(bool(pred_sql) and validate_response_format(solution_str)) - - if not pred_sql: - return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} - - alternative_answers = extra_info.get("alternative_answers") - if alternative_answers and len(alternative_answers) > 0: - gold_sqls = [str(s).strip() for s in alternative_answers if s and str(s).strip()] - else: - gold_sqls = [ground_truth] if ground_truth else [] - - if not gold_sqls: - return {"score": 0.0, "format_correct": format_correct, "execution_success": 0.0} - - reward, execution_success = _compare_results(db_path, pred_sql, gold_sqls) - - if format_correct: - reward = max(reward, FORMAT_REWARD_BONUS) - - return { - "score": reward, - "format_correct": format_correct, - "execution_success": float(execution_success), - } diff --git a/examples/arctic_rl/debug_at_gsm8k_grpo.sh b/examples/arctic_rl/debug_at_gsm8k_grpo.sh deleted file mode 100755 index 4845ff784a6..00000000000 --- a/examples/arctic_rl/debug_at_gsm8k_grpo.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -set -x -# enable if HF hub misbehaves/times out (assuming you have already cached the models locally) -export HF_HUB_OFFLINE=1 - -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -# we want to make sure this runs on non-gpu client -export CUDA_VISIBLE_DEVICES= - -export USE_ARCTIC_ZORRO=1 - -# BSZ=1024 -BSZ=2 -MBS=2 -UBS=2 -ROLL_N=4 -MAX_STEPS=4 -# LR=0 -LR=1e-6 -LOGGER=console -# LOGGER="['console','wandb']" -USE_KL_LOSS=True -# USE_KL_LOSS=False -REMOVE_PADDING=True -# REMOVE_PADDING=False -MODEL="Qwen/Qwen3-0.6B" -# STRATEGY="fsdp" -STRATEGY="fsdp2" -PYTHONUNBUFFERED=1 -HYDRA_FULL_ERROR=1 -USE_LEGACY_WORKER_IMPL=disable -USE_ARCTIC_RL=True -NGPU_PER_NODE=1 -ROLLOUT_NAME=arctic # entry point into ArcticRL -experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_at${USE_ARCTIC_RL}" - -gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) -if [[ $gpu_name == *"H200"* ]]; then - echo "Running on Hopper" - flash_attention_v=flash_attention_3 -elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then - echo "Running on Blackwell" - flash_attention_v=flash_attention_2 -else - echo "Running on unknown: $gpu_name; don't know which FA version to use" -fi - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=/code/shared/gsm8k/train.parquet \ - data.val_files=/code/shared/gsm8k/test.parquet \ - data.train_batch_size=${BSZ} \ - data.max_prompt_length=64 \ - data.max_response_length=16 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.shuffle=False \ - reward.num_workers=1 \ - actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.model.path=${MODEL} \ - actor_rollout_ref.actor.optim.lr=${LR} \ - actor_rollout_ref.model.use_remove_padding=${REMOVE_PADDING} \ - actor_rollout_ref.actor.ppo_mini_batch_size=${MBS} \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS} \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.actor.strategy=${STRATEGY} \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.enforce_eager=True \ - actor_rollout_ref.rollout.n=${ROLL_N} \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${UBS} \ - actor_rollout_ref.ref.fsdp_config.param_offload=False \ - actor_rollout_ref.ref.strategy=${STRATEGY} \ - algorithm.use_kl_in_reward=False \ - trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ - trainer.use_arctic_rl=${USE_ARCTIC_RL} \ - trainer.critic_warmup=0 \ - trainer.logger=${LOGGER} \ - trainer.experiment_name=${experiment_name} \ - trainer.project_name='verl_arctic_grpo_gsm8k' \ - trainer.val_before_train=False \ - trainer.n_gpus_per_node=${NGPU_PER_NODE} \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_training_steps=${MAX_STEPS} \ - trainer.total_epochs=15 $@ 2>&1 | tee ${experiment_name}.log - - # trainer.total_training_steps=${MAX_STEPS} \ diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh deleted file mode 100755 index 098169ef59e..00000000000 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_no.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash - -set -x - -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -export HF_HUB_OFFLINE=1 -export HF_HOME=/checkpoint/huggingface -export USE_ARCTIC_TRAINING_CLIENT=1 -# we want to make sure this runs on non-gpu client -export CUDA_VISIBLE_DEVICES= - -# BSZ=1024 -# UBS=32 -# ROLL_N=5 -# MAX_STEPS=100 -# PROMPT_LENGTH=512 -# RESPONSE_LENGTH=1024 - -# BSZ=4 -# UBS=2 -# ROLL_N=5 -# MAX_STEPS=4 -# PROMPT_LENGTH=512 -# RESPONSE_LENGTH=1024 - -BSZ=4 -UBS=2 -ROLL_N=2 -MAX_STEPS=1 -PROMPT_LENGTH=64 -RESPONSE_LENGTH=16 - -# LR=0 -LR=1e-6 - -LOGGER=console -# LOGGER="['console','wandb']" -USE_KL_LOSS=True -# USE_KL_LOSS=False -# REMOVE_PADDING=True -REMOVE_PADDING=False -MODEL="Qwen/Qwen3-0.6B" -# STRATEGY="fsdp" -STRATEGY="fsdp2" -USE_LEGACY_WORKER_IMPL=disable -NGPU_PER_NODE=1 -ROLLOUT_NAME=arctic # entry point into ArcticRL -USE_ARCTIC_RL=True -USE_ARCTIC_ZORRO=False -# COLOCATE=True -COLOCATE=False -experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_zorro${USE_ARCTIC_ZORRO}" - -gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) -if [[ $gpu_name == *"H200"* ]]; then - echo "Running on Hopper" - flash_attention_v=flash_attention_3 -elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then - echo "Running on Blackwell" - flash_attention_v=flash_attention_2 -else - echo "Running on unknown: $gpu_name; don't know which FA version to use" -fi - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=/code/shared/gsm8k/train.parquet \ - data.val_files=/code/shared/gsm8k/test.parquet \ - data.train_batch_size=$BSZ \ - data.max_prompt_length=$PROMPT_LENGTH \ - data.max_response_length=$RESPONSE_LENGTH \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.shuffle=False \ - +data.seed=42 \ - actor_rollout_ref.actor.data_loader_seed=42 \ - reward.num_workers=1 \ - actor_rollout_ref.rollout.agent.num_workers=4 \ - actor_rollout_ref.model.path=$MODEL \ - actor_rollout_ref.actor.optim.lr=$LR \ - actor_rollout_ref.model.use_remove_padding=$REMOVE_PADDING \ - actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=$UBS \ - actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.actor.strategy=$STRATEGY \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=$UBS \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.enforce_eager=True \ - actor_rollout_ref.rollout.n=$ROLL_N \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=$UBS \ - actor_rollout_ref.ref.fsdp_config.param_offload=False \ - actor_rollout_ref.ref.strategy=$STRATEGY \ - algorithm.use_kl_in_reward=False \ - trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ - trainer.use_arctic_rl=$USE_ARCTIC_RL \ - arctic_rl.colocate=$COLOCATE \ - arctic_rl.training_gpus=1\ - arctic_rl.sampling_gpus=2\ - arctic_rl.log_prob_gpus=1\ - arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ - trainer.critic_warmup=0 \ - trainer.logger=$LOGGER \ - trainer.experiment_name=$experiment_name \ - trainer.project_name='verl_arctic_grpo_gsm8k' \ - trainer.val_before_train=False \ - trainer.n_gpus_per_node=$NGPU_PER_NODE \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_training_steps=$MAX_STEPS \ - trainer.total_epochs=15 $@ 2>&1 | tee $experiment_name.log - - # trainer.total_training_steps=$MAX_STEPS \ diff --git a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh b/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh deleted file mode 100755 index 892cef3e141..00000000000 --- a/examples/arctic_rl/run_arctic_gsm8k_grpo_zorro_yes.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash - -set -x - -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -export HF_HUB_OFFLINE=1 -export HF_HOME=/checkpoint/huggingface -export USE_ARCTIC_TRAINING_CLIENT=1 -# we want to make sure this runs on non-gpu client -export CUDA_VISIBLE_DEVICES= - -BSZ=16 -UBS=16 -ROLL_N=5 -MAX_STEPS=40 -PROMPT_LENGTH=512 -RESPONSE_LENGTH=1024 - -# BSZ=1 -# UBS=1 -# ROLL_N=16 -# MAX_STEPS=4 -# PROMPT_LENGTH=1024 -# RESPONSE_LENGTH=2048 - -# BSZ=2 -# UBS=2 -# ROLL_N=4 -# MAX_STEPS=4 -# PROMPT_LENGTH=64 -# RESPONSE_LENGTH=512 - -# LR=0 -LR=1e-6 - -#LOGGER=console -LOGGER="['console','wandb']" -# USE_KL_LOSS=True -USE_KL_LOSS=False -# REMOVE_PADDING=True -REMOVE_PADDING=False -MODEL="Qwen/Qwen3-0.6B" -# STRATEGY="fsdp" -STRATEGY="fsdp2" -USE_LEGACY_WORKER_IMPL=disable -NGPU_PER_NODE=1 -ROLLOUT_NAME=arctic # entry point into ArcticRL -USE_ARCTIC_RL=True -USE_ARCTIC_ZORRO=True -COLOCATE=False -experiment_name="qwen3-0.6B_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_zorro${USE_ARCTIC_ZORRO}" - -gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) -if [[ $gpu_name == *"H200"* ]]; then - echo "Running on Hopper" - flash_attention_v=flash_attention_3 -elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then - echo "Running on Blackwell" - flash_attention_v=flash_attention_2 -else - echo "Running on unknown: $gpu_name; don't know which FA version to use" -fi - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=/code/shared/gsm8k/train.parquet \ - data.val_files=/code/shared/gsm8k/test.parquet \ - data.train_batch_size=$BSZ \ - data.max_prompt_length=$PROMPT_LENGTH \ - data.max_response_length=$RESPONSE_LENGTH \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.shuffle=False \ - +data.seed=42 \ - actor_rollout_ref.actor.data_loader_seed=42 \ - reward.num_workers=1 \ - actor_rollout_ref.rollout.agent.num_workers=4 \ - actor_rollout_ref.model.path=$MODEL \ - actor_rollout_ref.actor.optim.lr=$LR \ - actor_rollout_ref.model.use_remove_padding=$REMOVE_PADDING \ - actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=$UBS \ - actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.actor.strategy=$STRATEGY \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=$UBS \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.enforce_eager=True \ - actor_rollout_ref.rollout.n=$ROLL_N \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=$UBS \ - actor_rollout_ref.ref.fsdp_config.param_offload=False \ - actor_rollout_ref.ref.strategy=$STRATEGY \ - algorithm.use_kl_in_reward=False \ - trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ - trainer.use_arctic_rl=$USE_ARCTIC_RL \ - arctic_rl.colocate=$COLOCATE \ - arctic_rl.training_gpus=1\ - arctic_rl.sampling_gpus=2\ - arctic_rl.log_prob_gpus=1\ - arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ - trainer.critic_warmup=0 \ - trainer.logger=$LOGGER \ - trainer.experiment_name=$experiment_name \ - trainer.project_name=arctic_rl_bird_sql \ - trainer.val_before_train=False \ - trainer.n_gpus_per_node=$NGPU_PER_NODE \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_training_steps=$MAX_STEPS \ - trainer.total_epochs=15 \ - "$@" 2>&1 | tee $experiment_name.log - diff --git a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh deleted file mode 100755 index 39462bb84bc..00000000000 --- a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_no.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# GRPO training for Qwen3-1.7B on BIRD SQL dataset -# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) -# -# 1 node, 8 GPUs -# -# Prerequisites: -# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py -# 2. pip install func_timeout - -set -x - -experiment_name='qwen3_1.7b_bird_grpo_zorro_no' - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH}" -MAX_STEPS=4 -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -export HF_HUB_OFFLINE=1 -export HF_HOME=/checkpoint/huggingface -export USE_ARCTIC_TRAINING_CLIENT=1 -export CUDA_VISIBLE_DEVICES= -USE_ARCTIC_RL=True # entry point into ArcticRL - -USE_LEGACY_WORKER_IMPL=disable -ROLLOUT_NAME=arctic -NUM_AGENT_WORKERS=1 -NGPU_PER_NODE=1 - -# BSZ=128 -# PROMPT_LEN=16384 -# RESPONSE_LEN=4096 -# ROLL_N=16 - -BSZ=16 -UBS=4 -PROMPT_LEN=16384 -RESPONSE_LEN=4096 -ROLL_N=16 - -# LOGGER=console -LOGGER="['console','wandb']" - -gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) -if [[ $gpu_name == *"H200"* ]]; then - echo "Running on Hopper" - flash_attention_v=flash_attention_3 -elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then - echo "Running on Blackwell" - flash_attention_v=flash_attention_2 -else - echo "Running on unknown: $gpu_name; don't know which FA version to use" -fi - -# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" -# TRAIN_FILES="${DATA_DIR}/train.parquet" -# VAL_FILES="${DATA_DIR}/val.parquet" - -DATA_DIR="/code/shared/open-source-text2sql" -TRAIN_FILES="${DATA_DIR}/train.parquet" -VAL_FILES="${DATA_DIR}/val.parquet" - - -# LOG_PROBS=True -LOG_PROBS=False - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - algorithm.norm_adv_by_std_in_grpo=True \ - algorithm.use_kl_in_reward=False \ - algorithm.kl_ctrl.kl_coef=0.001 \ - data.train_files=${TRAIN_FILES} \ - data.val_files=${VAL_FILES} \ - data.train_batch_size=${BSZ} \ - data.max_prompt_length=${PROMPT_LEN} \ - data.max_response_length=${RESPONSE_LEN} \ - data.filter_overlong_prompts=True \ - data.filter_overlong_prompts_workers=1 \ - data.truncation=left \ - actor_rollout_ref.model.path=Qwen/Qwen3-1.7B \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.model.use_liger=True \ - actor_rollout_ref.actor.strategy=fsdp2 \ - actor_rollout_ref.actor.use_torch_compile=True \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=${BSZ} \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.actor.kl_loss_coef=0.0 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.optim.lr=5e-6 \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ - actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ - actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=${ROLL_N} \ - actor_rollout_ref.rollout.temperature=1.0 \ - actor_rollout_ref.rollout.top_p=1.0 \ - actor_rollout_ref.rollout.calculate_log_probs=${LOG_PROBS} \ - actor_rollout_ref.rollout.enforce_eager=False \ - actor_rollout_ref.rollout.max_num_seqs=256 \ - actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ - actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ - actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ - actor_rollout_ref.rollout.val_kwargs.temperature=0 \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.rollout.val_kwargs.do_sample=False \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - actor_rollout_ref.nccl_timeout=1800 \ - trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ - trainer.use_arctic_rl=${USE_ARCTIC_RL} \ - trainer.balance_batch=False \ - trainer.default_local_dir=/data-fast/sql-rl/${experiment_name} \ - trainer.logger=${LOGGER} \ - trainer.project_name=arctic_rl_bird_sql \ - trainer.experiment_name=${experiment_name} \ - trainer.n_gpus_per_node=${NGPU_PER_NODE} \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_epochs=10 \ - trainer.val_before_train=False \ - custom_reward_function.path="${SCRIPT_DIR}/bird_reward.py" \ - custom_reward_function.name=compute_score \ - trainer.total_training_steps=${MAX_STEPS} \ - "$@" 2>&1 | tee ${experiment_name}.log diff --git a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh b/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh deleted file mode 100755 index 6bc994c915a..00000000000 --- a/examples/arctic_rl/run_arctic_qwen3_1.7b_bird_grpo_zorro_yes.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash -# GRPO training for Qwen3-1.7B on BIRD SQL dataset -# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) -# -# 1 node, 8 GPUs -# -# Prerequisites: -# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py -# 2. pip install func_timeout - -set -x - -experiment_name='qwen3_1.7b_bird_grpo_zorro_yes' - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH}" -MAX_STEPS=4 -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -export HF_HUB_OFFLINE=1 -export HF_HOME=/checkpoint/huggingface -export USE_ARCTIC_TRAINING_CLIENT=1 -export CUDA_VISIBLE_DEVICES= -USE_ARCTIC_RL=True # entry point into ArcticRL - -USE_LEGACY_WORKER_IMPL=disable -ROLLOUT_NAME=arctic -NUM_AGENT_WORKERS=1 -NGPU_PER_NODE=1 - -# BSZ=128 -# PROMPT_LEN=16384 -# RESPONSE_LEN=4096 -# ROLL_N=16 - -BSZ=2 -PROMPT_LEN=16384 -RESPONSE_LEN=4096 -ROLL_N=2 -# LOGGER=console -LOGGER="['console','wandb']" - -gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) -if [[ $gpu_name == *"H200"* ]]; then - echo "Running on Hopper" - flash_attention_v=flash_attention_3 -elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then - echo "Running on Blackwell" - flash_attention_v=flash_attention_2 -else - echo "Running on unknown: $gpu_name; don't know which FA version to use" -fi - -# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" -# TRAIN_FILES="${DATA_DIR}/train.parquet" -# VAL_FILES="${DATA_DIR}/val.parquet" - -DATA_DIR="/code/shared/open-source-text2sql" -TRAIN_FILES="${DATA_DIR}/train.parquet" -VAL_FILES="${DATA_DIR}/val.parquet" - -# LOG_PROBS=True -LOG_PROBS=False - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - algorithm.norm_adv_by_std_in_grpo=True \ - algorithm.use_kl_in_reward=False \ - algorithm.kl_ctrl.kl_coef=0.001 \ - data.train_files=${TRAIN_FILES} \ - data.val_files=${VAL_FILES} \ - data.train_batch_size=${BSZ} \ - data.max_prompt_length=${PROMPT_LEN} \ - data.max_response_length=${RESPONSE_LEN} \ - data.filter_overlong_prompts=True \ - data.filter_overlong_prompts_workers=1 \ - data.truncation=left \ - actor_rollout_ref.model.path=Qwen/Qwen3-1.7B \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.model.use_liger=True \ - actor_rollout_ref.actor.strategy=fsdp2 \ - actor_rollout_ref.actor.use_torch_compile=True \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=${BSZ} \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.actor.kl_loss_coef=0.0 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.optim.lr=5e-6 \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ - actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ - actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=${ROLL_N} \ - actor_rollout_ref.rollout.temperature=1.0 \ - actor_rollout_ref.rollout.top_p=1.0 \ - actor_rollout_ref.rollout.calculate_log_probs=${LOG_PROBS} \ - actor_rollout_ref.rollout.enforce_eager=False \ - actor_rollout_ref.rollout.max_num_seqs=256 \ - actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ - actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ - actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ - actor_rollout_ref.rollout.val_kwargs.temperature=0 \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.rollout.val_kwargs.do_sample=False \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - actor_rollout_ref.nccl_timeout=1800 \ - trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ - trainer.use_arctic_rl=${USE_ARCTIC_RL} \ - trainer.balance_batch=False \ - trainer.default_local_dir=/data-fast/sql-rl/${experiment_name} \ - trainer.logger=${LOGGER} \ - trainer.project_name=arctic_rl_bird_sql \ - trainer.experiment_name=${experiment_name} \ - trainer.n_gpus_per_node=${NGPU_PER_NODE} \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_epochs=10 \ - trainer.val_before_train=False \ - custom_reward_function.path="${SCRIPT_DIR}/bird_reward.py" \ - custom_reward_function.name=compute_score \ - trainer.total_training_steps=${MAX_STEPS} \ - "$@" 2>&1 | tee ${experiment_name}.log diff --git a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh deleted file mode 100755 index 8c17d88740d..00000000000 --- a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -# GRPO training for Qwen3-1.7B on BIRD SQL dataset -# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) -# -# 1 node, 8 GPUs -# -# Prerequisites: -# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py -# 2. pip install func_timeout - -set -x - -SCRIPT_DIR="$(cd "$(dirname "$BASH_SOURCE[0]")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" - -export PYTHONPATH="$REPO_ROOT:$PYTHONPATH" - -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -export HF_HUB_OFFLINE=1 -export HF_HOME=/checkpoint/huggingface -export USE_ARCTIC_TRAINING_CLIENT=1 # entry point into ArcticRL -export CUDA_VISIBLE_DEVICES= - -USE_LEGACY_WORKER_IMPL=disable -ROLLOUT_NAME=arctic -NGPU_PER_NODE=1 - -USE_ARCTIC_RL=True -USE_ARCTIC_ZORRO=True -COLOCATE=False - -# BSZ=32 -# ROLL_N=16 -BSZ=4 -ROLL_N=4 -MAX_STEPS=10 -PROMPT_LEN=16384 -RESPONSE_LEN=4096 - -LOGGER=console -# LOGGER="['console','wandb']" - -#MODEL_SHORT=Qwen3-1.7B -MODEL_SHORT=Qwen3-0.6B - -MODEL=Qwen/$MODEL_SHORT - -experiment_name="bird_grpo_${MODEL_SHORT}_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_arl_zorro_yes" - -gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) -if [[ $gpu_name == *"H200"* ]]; then - echo "Running on Hopper" - flash_attention_v=flash_attention_3 -elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then - echo "Running on Blackwell" - flash_attention_v=flash_attention_2 -else - echo "Running on unknown: $gpu_name; don't know which FA version to use" -fi - -# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" -# TRAIN_FILES="$DATA_DIR/train.parquet" -# VAL_FILES="$DATA_DIR/val.parquet" - - -DATA_DIR="/code/shared/open-source-text2sql" -#TRAIN_FILES="$DATA_DIR/train.parquet" -#TRAIN_FILES="$DATA_DIR/train-1000.parquet" -TRAIN_FILES="$DATA_DIR/train-100.parquet" -VAL_FILES="$DATA_DIR/val.parquet" - -# LOG_PROBS=True -LOG_PROBS=False - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - algorithm.norm_adv_by_std_in_grpo=True \ - algorithm.use_kl_in_reward=False \ - algorithm.kl_ctrl.kl_coef=0.001 \ - data.train_files=$TRAIN_FILES \ - data.val_files=$VAL_FILES \ - data.train_batch_size=$BSZ \ - data.max_prompt_length=$PROMPT_LEN \ - data.max_response_length=$RESPONSE_LEN \ - data.filter_overlong_prompts=True \ - data.filter_overlong_prompts_workers=1 \ - data.truncation=left \ - actor_rollout_ref.model.path=$MODEL \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.model.use_liger=True \ - actor_rollout_ref.actor.strategy=fsdp2 \ - actor_rollout_ref.actor.use_torch_compile=True \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.actor.kl_loss_coef=0.0 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.optim.lr=5e-6 \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ - actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ - actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=$ROLL_N \ - actor_rollout_ref.rollout.temperature=1.0 \ - actor_rollout_ref.rollout.top_p=1.0 \ - actor_rollout_ref.rollout.calculate_log_probs=$LOG_PROBS \ - actor_rollout_ref.rollout.enforce_eager=False \ - actor_rollout_ref.rollout.max_num_seqs=256 \ - actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ - actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ - actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ - actor_rollout_ref.rollout.val_kwargs.temperature=0 \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.rollout.val_kwargs.do_sample=False \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - actor_rollout_ref.nccl_timeout=1800 \ - trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ - trainer.use_arctic_rl=$USE_ARCTIC_RL \ - trainer.balance_batch=False \ - trainer.default_local_dir=/data-fast/sql-rl/$experiment_name \ - trainer.logger=$LOGGER \ - trainer.project_name=arctic_rl_bird_sql \ - trainer.experiment_name=$experiment_name \ - trainer.n_gpus_per_node=$NGPU_PER_NODE \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_epochs=10 \ - trainer.val_before_train=False \ - custom_reward_function.path="$SCRIPT_DIR/bird_reward.py" \ - custom_reward_function.name=compute_score \ - trainer.total_training_steps=$MAX_STEPS \ - arctic_rl.colocate=$COLOCATE \ - arctic_rl.training_gpus=2\ - arctic_rl.sampling_gpus=4\ - arctic_rl.log_prob_gpus=2\ - arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ - "$@" 2>&1 | tee $experiment_name.log diff --git a/examples/arctic_rl/run_bird_grpo_baseline.sh b/examples/arctic_rl/run_bird_grpo_baseline.sh deleted file mode 100755 index 8d160368473..00000000000 --- a/examples/arctic_rl/run_bird_grpo_baseline.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -# GRPO training for Qwen3-1.7B on BIRD SQL dataset -# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) -# -# 1 node, 8 GPUs -# -# Prerequisites: -# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py -# 2. pip install func_timeout - -set -x - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" - -export PYTHONPATH="$REPO_ROOT:$PYTHONPATH" -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -export HF_HUB_OFFLINE=1 -export HF_HOME=/checkpoint/huggingface - -USE_LEGACY_WORKER_IMPL=disable -ROLLOUT_NAME=vllm -NGPU_PER_NODE=1 - -BSZ=32 -ROLL_N=16 -MAX_STEPS=10 -PROMPT_LEN=16384 -RESPONSE_LEN=4096 - -#LOGGER=console -LOGGER="['console','wandb']" - -#MODEL_SHORT=Qwen3-1.7B -MODEL_SHORT=Qwen3-0.6B - -MODEL=Qwen/$MODEL_SHORT - -experiment_name="bird_grpo_${MODEL_SHORT}_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_baseline" - -gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) -if [[ $gpu_name == *"H200"* ]]; then - echo "Running on Hopper" - flash_attention_v=flash_attention_3 -elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then - echo "Running on Blackwell" - flash_attention_v=flash_attention_2 -else - echo "Running on unknown: $gpu_name; don't know which FA version to use" -fi - -# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" -# TRAIN_FILES="$DATA_DIR/train.parquet" -# VAL_FILES="$DATA_DIR/val.parquet" - - -DATA_DIR="/code/shared/open-source-text2sql" -#TRAIN_FILES="$DATA_DIR/train.parquet" -#TRAIN_FILES="$DATA_DIR/train-1000.parquet" -TRAIN_FILES="$DATA_DIR/train-100.parquet" -VAL_FILES="$DATA_DIR/val.parquet" - -# LOG_PROBS=True -LOG_PROBS=False - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - algorithm.norm_adv_by_std_in_grpo=True \ - algorithm.use_kl_in_reward=False \ - algorithm.kl_ctrl.kl_coef=0.001 \ - data.train_files=$TRAIN_FILES \ - data.val_files=$VAL_FILES \ - data.train_batch_size=$BSZ \ - data.max_prompt_length=$PROMPT_LEN \ - data.max_response_length=$RESPONSE_LEN \ - data.filter_overlong_prompts=True \ - data.filter_overlong_prompts_workers=1 \ - data.truncation=left \ - actor_rollout_ref.model.path=$MODEL \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.model.use_liger=True \ - actor_rollout_ref.actor.strategy=fsdp2 \ - actor_rollout_ref.actor.use_torch_compile=True \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.actor.kl_loss_coef=0.0 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.optim.lr=5e-6 \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ - actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ - actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=$ROLL_N \ - actor_rollout_ref.rollout.temperature=1.0 \ - actor_rollout_ref.rollout.top_p=1.0 \ - actor_rollout_ref.rollout.calculate_log_probs=$LOG_PROBS \ - actor_rollout_ref.rollout.enforce_eager=False \ - actor_rollout_ref.rollout.max_num_seqs=256 \ - actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ - actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ - actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ - actor_rollout_ref.rollout.val_kwargs.temperature=0 \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.rollout.val_kwargs.do_sample=False \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - actor_rollout_ref.nccl_timeout=1800 \ - trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ - trainer.balance_batch=False \ - trainer.default_local_dir=/data-fast/sql-rl/$experiment_name \ - trainer.logger=$LOGGER \ - trainer.project_name=arctic_rl_bird_sql \ - trainer.experiment_name=$experiment_name \ - trainer.n_gpus_per_node=$NGPU_PER_NODE \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_epochs=10 \ - trainer.val_before_train=False \ - custom_reward_function.path="$SCRIPT_DIR/bird_reward.py" \ - custom_reward_function.name=compute_score \ - trainer.total_training_steps=$MAX_STEPS \ - "$@" 2>&1 | tee $experiment_name.log diff --git a/examples/arctic_rl/run_gsm8k_grpo.sh b/examples/arctic_rl/run_gsm8k_grpo.sh deleted file mode 100755 index 3e4d53831f1..00000000000 --- a/examples/arctic_rl/run_gsm8k_grpo.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -set -x - -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 - -# BSZ=1024 -# UBS=32 -ROLL_N=5 -# MAX_STEPS=100 -PROMPT_LENGTH=512 -RESPONSE_LENGTH=1024 -BSZ=8 -UBS=2 -# ROLL_N=2 -MAX_STEPS=4 -# PROMPT_LENGTH=64 -# RESPONSE_LENGTH=512 - -# LR=0 -LR=1e-6 - -# LOGGER=console -LOGGER="['console','wandb']" -# USE_KL_LOSS=True -USE_KL_LOSS=False -# REMOVE_PADDING=True -REMOVE_PADDING=False -MODEL="Qwen/Qwen3-0.6B" -# STRATEGY="fsdp" -STRATEGY="fsdp2" -PYTHONUNBUFFERED=1 -HYDRA_FULL_ERROR=1 -USE_LEGACY_WORKER_IMPL=disable -NGPU_PER_NODE=4 -ROLLOUT_NAME=vllm - -experiment_name="qwen3-0.6B_ngpu$NGPU_PER_NODE_gbs$BSZ_rolln$ROLL_N_baseline" -gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) -if [[ $gpu_name == *"H200"* ]]; then - echo "Running on Hopper" - flash_attention_v=flash_attention_3 -elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then - echo "Running on Blackwell" - flash_attention_v=flash_attention_2 -else - echo "Running on unknown: $gpu_name; don't know which FA version to use" -fi - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=/code/shared/gsm8k/train.parquet \ - data.val_files=/code/shared/gsm8k/test.parquet \ - data.train_batch_size=$BSZ \ - data.max_prompt_length=$PROMPT_LENGTH \ - data.max_response_length=$RESPONSE_LENGTH \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.shuffle=False \ - +data.seed=42 \ - actor_rollout_ref.actor.data_loader_seed=42 \ - reward.num_workers=1 \ - actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.model.path=$MODEL \ - actor_rollout_ref.actor.optim.lr=$LR \ - actor_rollout_ref.model.use_remove_padding=$REMOVE_PADDING \ - actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=$UBS \ - actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.actor.strategy=$STRATEGY \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=$UBS \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.enforce_eager=True \ - actor_rollout_ref.rollout.n=$ROLL_N \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=$UBS \ - actor_rollout_ref.ref.fsdp_config.param_offload=False \ - actor_rollout_ref.ref.strategy=$STRATEGY \ - algorithm.use_kl_in_reward=False \ - trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ - trainer.critic_warmup=0 \ - trainer.logger=$LOGGER \ - trainer.experiment_name=$experiment_name \ - trainer.project_name='verl_arctic_grpo_gsm8k' \ - trainer.val_before_train=False \ - trainer.n_gpus_per_node=$NGPU_PER_NODE \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_training_steps=$MAX_STEPS \ - trainer.total_epochs=15 $@ 2>&1 | tee $experiment_name.log - - # trainer.total_training_steps=$MAX_STEPS \ diff --git a/examples/arctic_rl/run_qwen3_1.7b_bird_grpo_baseline.sh b/examples/arctic_rl/run_qwen3_1.7b_bird_grpo_baseline.sh deleted file mode 100755 index a555ffdf523..00000000000 --- a/examples/arctic_rl/run_qwen3_1.7b_bird_grpo_baseline.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash -# GRPO training for Qwen3-1.7B on BIRD SQL dataset -# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) -# -# 1 node, 8 GPUs -# -# Prerequisites: -# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py -# 2. pip install func_timeout - -set -x - -experiment_name='qwen3_1.7b_bird_grpo_baseline' - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH}" -MAX_STEPS=4 -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -export HF_HUB_OFFLINE=1 -USE_LEGACY_WORKER_IMPL=disable -ROLLOUT_NAME=vllm -NUM_AGENT_WORKERS=1 -NGPU_PER_NODE=1 - -# BSZ=128 -# PROMPT_LEN=16384 -# RESPONSE_LEN=4096 -# ROLL_N=16 - -BSZ=2 -PROMPT_LEN=16384 -RESPONSE_LEN=4096 -ROLL_N=2 - -# LOGGER=console -LOGGER="['console','wandb']" - -gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) -if [[ $gpu_name == *"H200"* ]]; then - echo "Running on Hopper" - flash_attention_v=flash_attention_3 -elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then - echo "Running on Blackwell" - flash_attention_v=flash_attention_2 -else - echo "Running on unknown: $gpu_name; don't know which FA version to use" -fi - -# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" -# TRAIN_FILES="${DATA_DIR}/train.parquet" -# VAL_FILES="${DATA_DIR}/val.parquet" - - -DATA_DIR="/code/shared/open-source-text2sql" -TRAIN_FILES="${DATA_DIR}/train.parquet" -VAL_FILES="${DATA_DIR}/val.parquet" - - -# LOG_PROBS=True -LOG_PROBS=False - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - algorithm.norm_adv_by_std_in_grpo=True \ - algorithm.use_kl_in_reward=False \ - algorithm.kl_ctrl.kl_coef=0.001 \ - data.train_files=${TRAIN_FILES} \ - data.val_files=${VAL_FILES} \ - data.train_batch_size=${BSZ} \ - data.max_prompt_length=${PROMPT_LEN} \ - data.max_response_length=${RESPONSE_LEN} \ - data.filter_overlong_prompts=True \ - data.filter_overlong_prompts_workers=1 \ - data.truncation=left \ - actor_rollout_ref.model.path=Qwen/Qwen3-1.7B \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.model.use_liger=True \ - actor_rollout_ref.actor.strategy=fsdp2 \ - actor_rollout_ref.actor.use_torch_compile=True \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=${BSZ} \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.actor.kl_loss_coef=0.0 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.optim.lr=5e-6 \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ - actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=${ROLLOUT_NAME} \ - actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=${ROLL_N} \ - actor_rollout_ref.rollout.temperature=1.0 \ - actor_rollout_ref.rollout.top_p=1.0 \ - actor_rollout_ref.rollout.calculate_log_probs=${LOG_PROBS} \ - actor_rollout_ref.rollout.enforce_eager=False \ - actor_rollout_ref.rollout.max_num_seqs=256 \ - actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ - actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ - actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ - actor_rollout_ref.rollout.val_kwargs.temperature=0 \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.rollout.val_kwargs.do_sample=False \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - actor_rollout_ref.nccl_timeout=1800 \ - trainer.use_legacy_worker_impl=${USE_LEGACY_WORKER_IMPL} \ - trainer.balance_batch=False \ - trainer.default_local_dir=/data-fast/sql-rl/${experiment_name} \ - trainer.logger=${LOGGER} \ - trainer.project_name=arctic_rl_bird_sql \ - trainer.experiment_name=${experiment_name} \ - trainer.n_gpus_per_node=${NGPU_PER_NODE} \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_epochs=10 \ - trainer.val_before_train=False \ - custom_reward_function.path="${SCRIPT_DIR}/bird_reward.py" \ - custom_reward_function.name=compute_score \ - trainer.total_training_steps=${MAX_STEPS} \ - "$@" 2>&1 | tee ${experiment_name}.log diff --git a/verl/workers/utils/padding.py b/verl/workers/utils/padding.py index f6883fbc748..16242e7731f 100644 --- a/verl/workers/utils/padding.py +++ b/verl/workers/utils/padding.py @@ -94,7 +94,6 @@ def no_padding_2_padding(tensor: torch.Tensor, data: TensorDict) -> torch.Tensor Returns: tensor: sliced response tensor of shape [bsz, max_response_len] """ - # print(f"{tensor.is_nested=}") values = tensor.values() if tensor.is_nested else tensor prompt_ids = data["prompts"] response_ids = data["responses"] From 6b9e257bc753e5328f8dba9603580ee7301db9ab Mon Sep 17 00:00:00 2001 From: tunji-ruwase_snow Date: Thu, 23 Apr 2026 22:02:55 +0000 Subject: [PATCH 53/58] Add bird run script --- examples/arctic_rl/bird_reward.py | 272 ++++++++++++++++++ .../arctic_rl/run_bird_grpo_arl_zorro_yes.sh | 149 ++++++++++ .../rollout/arctic_rollout/arctic_rollout.py | 2 +- 3 files changed, 422 insertions(+), 1 deletion(-) create mode 100644 examples/arctic_rl/bird_reward.py create mode 100755 examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh diff --git a/examples/arctic_rl/bird_reward.py b/examples/arctic_rl/bird_reward.py new file mode 100644 index 00000000000..92a0bec4a30 --- /dev/null +++ b/examples/arctic_rl/bird_reward.py @@ -0,0 +1,272 @@ +""" +SQL reward function for BIRD RL training, adapted from SnowflakeDialectSQLRewardManagerV6b. + +Uses SQLite execution instead of Snowflake. Compatible with verl's +custom_reward_function mechanism via compute_score(). + +Reward scheme (matching V6b non-semantic-model behavior): + 1.0 - Predicted SQL produces the same result set as gold SQL + 0.1 - Predicted SQL executes successfully but produces wrong results, + OR SQL was extracted successfully (format bonus) + 0.0 - No SQL extracted, SQL fails to execute, or timeout +""" + +import json +import re +import sqlite3 +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError +from functools import lru_cache + +SQL_TIMEOUT = 30 +DEFAULT_LIMIT_NUMBER = 5000 +FORMAT_REWARD_BONUS = 0.1 + + +# --------------------------------------------------------------------------- +# SQL extraction (mirrors V6b's extract_solution / _extract_sql_omnisql) +# --------------------------------------------------------------------------- + +def _extract_sql_omnisql(message: str) -> str: + """Extract SQL from ```sql ... ``` markdown blocks (last valid block).""" + pattern = r"```sql\s*(.*?)\s*```" + sql_blocks = re.findall(pattern, message, re.DOTALL) + for block in reversed(sql_blocks): + if len(block.strip()) > 6: + return block.strip() + return "" + + +def _extract_sql_generic_block(message: str) -> str: + """Extract SQL from generic ``` ... ``` blocks containing SELECT.""" + blocks = re.findall(r"```\s*(.*?)\s*```", message, re.DOTALL) + for block in reversed(blocks): + if "SELECT" in block.upper() and len(block.strip()) > 6: + return block.strip() + return "" + + +def _extract_sql_analyst(message: str) -> str: + """Extract SQL from ```json { "sql": "..." } ``` blocks.""" + block = re.search(r"```\s*json(.*?)```", message, re.DOTALL) + if block is None: + return "" + json_str = block.group(1) + idx_left = json_str.rfind("{") + idx_right = json_str.find("}") + if idx_left == -1 or idx_right == -1: + return "" + json_str = json_str[idx_left : idx_right + 1] + try: + return json.loads(json_str.replace("\\n", "\n").replace("\\'", "'"), strict=False).get("sql", "") + except Exception: + return "" + + +def _extract_sql_raw_select(message: str) -> str: + """Fallback: extract a raw SELECT statement.""" + match = re.search(r"(SELECT\s+.+?)(?:\n\n|$)", message, re.DOTALL | re.IGNORECASE) + if match: + return match.group(1).strip() + return "" + + +def extract_sql(response: str) -> str: + """Extract SQL from model response, following V6b's extraction pipeline. + + 1. Split on to isolate the answer portion + 2. Try ```sql blocks + 3. Try generic ``` blocks with SELECT + 4. Try ```json blocks with {"sql": ...} + 5. Fallback to raw SELECT statement + """ + if "" in response: + answer_part = response.split("", 1)[1] + else: + answer_part = response + + sql = _extract_sql_omnisql(answer_part) + if sql: + return sql + + sql = _extract_sql_generic_block(answer_part) + if sql: + return sql + + sql = _extract_sql_analyst(answer_part) + if sql: + return sql + + return _extract_sql_raw_select(answer_part) + + +# --------------------------------------------------------------------------- +# Format validation (mirrors V6b's validate_response_structure) +# --------------------------------------------------------------------------- + +def validate_response_format(response: str) -> bool: + """Check that the response has exactly one ... pair, properly nested.""" + start_positions = [m.start() for m in re.finditer(r"", response)] + end_positions = [m.start() for m in re.finditer(r"", response)] + + if len(start_positions) != 1 or len(end_positions) != 1: + return False + return start_positions[0] < end_positions[0] + + +# --------------------------------------------------------------------------- +# LIMIT addition (mirrors V6b's _add_limit_to_query) +# --------------------------------------------------------------------------- + +def _add_limit_to_query(query: str, limit: int = DEFAULT_LIMIT_NUMBER) -> str: + """Add LIMIT clause if the query doesn't already have one.""" + if not query: + return query + upper = query.upper() + if "LIMIT " in upper or "LIMIT\n" in upper or "LIMIT\t" in upper: + return query + return query.rstrip().rstrip(";").rstrip() + f" LIMIT {limit};" + + +# --------------------------------------------------------------------------- +# SQLite execution and comparison +# --------------------------------------------------------------------------- + +def _execute_sql(db_path: str, sql: str, timeout: float = SQL_TIMEOUT) -> frozenset | Exception: + """Execute SQL against a SQLite database and return result as frozenset of row tuples.""" + try: + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5) + deadline = __import__("time").monotonic() + timeout + def _check_cancel(): + if __import__("time").monotonic() > deadline: + return 1 + return 0 + conn.set_progress_handler(_check_cancel, 1000) + cursor = conn.cursor() + cursor.execute(sql) + rows = cursor.fetchall() + conn.close() + return frozenset(rows) + except sqlite3.OperationalError as e: + if "interrupt" in str(e).lower(): + return TimeoutError(f"SQL execution exceeded {timeout}s") + return e + except Exception as e: + return e + + +def _execute_with_timeout(db_path: str, sql: str, timeout: float = SQL_TIMEOUT) -> frozenset | Exception: + """Execute SQL with a timeout using a thread pool. + + Uses shutdown(wait=False) to avoid blocking if the SQLite thread is stuck. + The SQLite progress handler provides cooperative cancellation. + """ + executor = ThreadPoolExecutor(max_workers=1) + future = executor.submit(_execute_sql, db_path, sql, timeout) + try: + return future.result(timeout=timeout + 2) + except FuturesTimeoutError: + return TimeoutError(f"SQL execution exceeded {timeout}s") + except Exception as e: + return e + finally: + executor.shutdown(wait=False) + + +def _compare_results( + db_path: str, + pred_sql: str, + gold_sqls: list[str], + timeout: float = SQL_TIMEOUT, +) -> tuple[float, bool]: + """Execute and compare predicted SQL against all gold SQLs. + + Returns (reward, execution_success) matching V6b's non-semantic-model logic: + - 1.0 if pred result == any gold result (frozenset match) + - 0.1 if pred executes but doesn't match any gold + - 0.0 if pred fails to execute + + Caches gold results within this call to avoid re-execution. + """ + pred_sql_limited = _add_limit_to_query(pred_sql) + pred_result = _execute_with_timeout(db_path, pred_sql_limited, timeout) + + if isinstance(pred_result, Exception): + return 0.0, False + + gold_cache: dict[str, frozenset | Exception] = {} + scores = [] + for gold_sql in gold_sqls: + if gold_sql not in gold_cache: + gold_sql_limited = _add_limit_to_query(gold_sql) + gold_cache[gold_sql] = _execute_with_timeout(db_path, gold_sql_limited, timeout) + gold_result = gold_cache[gold_sql] + + if isinstance(gold_result, Exception): + scores.append(0.0) + continue + + if pred_result == gold_result: + scores.append(1.0) + else: + scores.append(0.1) + + return (max(scores) if scores else 0.0), True + + +# --------------------------------------------------------------------------- +# Main entry point +# --------------------------------------------------------------------------- + +def compute_score(data_source, solution_str, ground_truth, extra_info=None, **kwargs): + """Compute reward score for SQL generation (verl custom_reward_function interface). + + Mirrors SnowflakeDialectSQLRewardManagerV6b logic with SQLite execution: + 1. Extract SQL from model response + 2. Validate response format ( tags) + 3. Execute predicted and gold SQL against SQLite + 4. Compare results (frozenset match) + 5. Apply format bonus + + Args: + data_source: Dataset identifier (e.g. "bird") + solution_str: Full model response text (decoded) + ground_truth: Gold SQL query string + extra_info: Dict with at minimum {"db_path": "/path/to/db.sqlite"}. + Optionally {"alternative_answers": [...]} for multiple gold SQLs. + + Returns: + dict with "score" (float), "format_correct" (float), "execution_success" (float) + """ + if extra_info is None: + return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} + + db_path = extra_info.get("db_path", "") + if not db_path: + return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} + + pred_sql = extract_sql(solution_str) + format_correct = float(bool(pred_sql) and validate_response_format(solution_str)) + + if not pred_sql: + return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} + + alternative_answers = extra_info.get("alternative_answers") + if alternative_answers and len(alternative_answers) > 0: + gold_sqls = [str(s).strip() for s in alternative_answers if s and str(s).strip()] + else: + gold_sqls = [ground_truth] if ground_truth else [] + + if not gold_sqls: + return {"score": 0.0, "format_correct": format_correct, "execution_success": 0.0} + + reward, execution_success = _compare_results(db_path, pred_sql, gold_sqls) + + if format_correct: + reward = max(reward, FORMAT_REWARD_BONUS) + + return { + "score": reward, + "format_correct": format_correct, + "execution_success": float(execution_success), + } diff --git a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh new file mode 100755 index 00000000000..8c17d88740d --- /dev/null +++ b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh @@ -0,0 +1,149 @@ +#!/bin/bash +# GRPO training for Qwen3-1.7B on BIRD SQL dataset +# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) +# +# 1 node, 8 GPUs +# +# Prerequisites: +# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py +# 2. pip install func_timeout + +set -x + +SCRIPT_DIR="$(cd "$(dirname "$BASH_SOURCE[0]")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +export PYTHONPATH="$REPO_ROOT:$PYTHONPATH" + +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +export HF_HUB_OFFLINE=1 +export HF_HOME=/checkpoint/huggingface +export USE_ARCTIC_TRAINING_CLIENT=1 # entry point into ArcticRL +export CUDA_VISIBLE_DEVICES= + +USE_LEGACY_WORKER_IMPL=disable +ROLLOUT_NAME=arctic +NGPU_PER_NODE=1 + +USE_ARCTIC_RL=True +USE_ARCTIC_ZORRO=True +COLOCATE=False + +# BSZ=32 +# ROLL_N=16 +BSZ=4 +ROLL_N=4 +MAX_STEPS=10 +PROMPT_LEN=16384 +RESPONSE_LEN=4096 + +LOGGER=console +# LOGGER="['console','wandb']" + +#MODEL_SHORT=Qwen3-1.7B +MODEL_SHORT=Qwen3-0.6B + +MODEL=Qwen/$MODEL_SHORT + +experiment_name="bird_grpo_${MODEL_SHORT}_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_arl_zorro_yes" + +gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) +if [[ $gpu_name == *"H200"* ]]; then + echo "Running on Hopper" + flash_attention_v=flash_attention_3 +elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then + echo "Running on Blackwell" + flash_attention_v=flash_attention_2 +else + echo "Running on unknown: $gpu_name; don't know which FA version to use" +fi + +# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" +# TRAIN_FILES="$DATA_DIR/train.parquet" +# VAL_FILES="$DATA_DIR/val.parquet" + + +DATA_DIR="/code/shared/open-source-text2sql" +#TRAIN_FILES="$DATA_DIR/train.parquet" +#TRAIN_FILES="$DATA_DIR/train-1000.parquet" +TRAIN_FILES="$DATA_DIR/train-100.parquet" +VAL_FILES="$DATA_DIR/val.parquet" + +# LOG_PROBS=True +LOG_PROBS=False + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + algorithm.norm_adv_by_std_in_grpo=True \ + algorithm.use_kl_in_reward=False \ + algorithm.kl_ctrl.kl_coef=0.001 \ + data.train_files=$TRAIN_FILES \ + data.val_files=$VAL_FILES \ + data.train_batch_size=$BSZ \ + data.max_prompt_length=$PROMPT_LEN \ + data.max_response_length=$RESPONSE_LEN \ + data.filter_overlong_prompts=True \ + data.filter_overlong_prompts_workers=1 \ + data.truncation=left \ + actor_rollout_ref.model.path=$MODEL \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ + actor_rollout_ref.model.use_liger=True \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.actor.use_torch_compile=True \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.optim.lr=5e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ + actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ + actor_rollout_ref.rollout.agent.num_workers=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=$ROLL_N \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.top_p=1.0 \ + actor_rollout_ref.rollout.calculate_log_probs=$LOG_PROBS \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.max_num_seqs=256 \ + actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ + actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ + actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ + actor_rollout_ref.rollout.val_kwargs.temperature=0 \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.rollout.val_kwargs.do_sample=False \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.nccl_timeout=1800 \ + trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ + trainer.use_arctic_rl=$USE_ARCTIC_RL \ + trainer.balance_batch=False \ + trainer.default_local_dir=/data-fast/sql-rl/$experiment_name \ + trainer.logger=$LOGGER \ + trainer.project_name=arctic_rl_bird_sql \ + trainer.experiment_name=$experiment_name \ + trainer.n_gpus_per_node=$NGPU_PER_NODE \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=10 \ + trainer.val_before_train=False \ + custom_reward_function.path="$SCRIPT_DIR/bird_reward.py" \ + custom_reward_function.name=compute_score \ + trainer.total_training_steps=$MAX_STEPS \ + arctic_rl.colocate=$COLOCATE \ + arctic_rl.training_gpus=2\ + arctic_rl.sampling_gpus=4\ + arctic_rl.log_prob_gpus=2\ + arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ + "$@" 2>&1 | tee $experiment_name.log diff --git a/verl/workers/rollout/arctic_rollout/arctic_rollout.py b/verl/workers/rollout/arctic_rollout/arctic_rollout.py index 6abcf86302b..2f187e2e95f 100644 --- a/verl/workers/rollout/arctic_rollout/arctic_rollout.py +++ b/verl/workers/rollout/arctic_rollout/arctic_rollout.py @@ -291,7 +291,7 @@ def __init__( super().__init__(replica_rank, config, model_config, gpus_per_node, is_reward_model) self.server_class = ray.remote(ArcticLLMServer) self.arctic_rl_client = kwargs.get("arctic_rl_client", None) - # assert self.arctic_rl_client is not None, "arctic_rl_client is required" + assert self.arctic_rl_client is not None, "arctic_rl_client is required" def rollout_worker_use_gpu(self) -> bool: From c723dc3d00146258ddef422863a9cc967223802b Mon Sep 17 00:00:00 2001 From: tunji-ruwase_snow Date: Thu, 23 Apr 2026 22:06:22 +0000 Subject: [PATCH 54/58] Enable weight sync --- verl/trainer/ppo/arctic_rl_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/verl/trainer/ppo/arctic_rl_client.py b/verl/trainer/ppo/arctic_rl_client.py index a6914680b3d..1ab23798905 100644 --- a/verl/trainer/ppo/arctic_rl_client.py +++ b/verl/trainer/ppo/arctic_rl_client.py @@ -199,10 +199,10 @@ def save_checkpoint(self): return response def update_weights(self): - return None # TODO: Implement this - # response = self._client.sync_weights() - # print(f"[ArcticRLClientWrapper] update_weights OUTPUT: {response.keys()=}") - # return response + # return None # TODO: Implement this + response = self._client.sync_weights() + print(f"[ArcticRLClientWrapper] update_weights OUTPUT: {response.keys()=}") + return response def destroy(self): if self._client is not None: From b8c13bbee7f75b6ddb09eae43b0aa253ce989133 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Tue, 28 Apr 2026 20:23:26 +0000 Subject: [PATCH 55/58] Remove --- examples/arctic_rl/bird_reward.py | 272 ------------------ .../arctic_rl/run_bird_grpo_arl_zorro_yes.sh | 149 ---------- 2 files changed, 421 deletions(-) delete mode 100644 examples/arctic_rl/bird_reward.py delete mode 100755 examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh diff --git a/examples/arctic_rl/bird_reward.py b/examples/arctic_rl/bird_reward.py deleted file mode 100644 index 92a0bec4a30..00000000000 --- a/examples/arctic_rl/bird_reward.py +++ /dev/null @@ -1,272 +0,0 @@ -""" -SQL reward function for BIRD RL training, adapted from SnowflakeDialectSQLRewardManagerV6b. - -Uses SQLite execution instead of Snowflake. Compatible with verl's -custom_reward_function mechanism via compute_score(). - -Reward scheme (matching V6b non-semantic-model behavior): - 1.0 - Predicted SQL produces the same result set as gold SQL - 0.1 - Predicted SQL executes successfully but produces wrong results, - OR SQL was extracted successfully (format bonus) - 0.0 - No SQL extracted, SQL fails to execute, or timeout -""" - -import json -import re -import sqlite3 -from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError -from functools import lru_cache - -SQL_TIMEOUT = 30 -DEFAULT_LIMIT_NUMBER = 5000 -FORMAT_REWARD_BONUS = 0.1 - - -# --------------------------------------------------------------------------- -# SQL extraction (mirrors V6b's extract_solution / _extract_sql_omnisql) -# --------------------------------------------------------------------------- - -def _extract_sql_omnisql(message: str) -> str: - """Extract SQL from ```sql ... ``` markdown blocks (last valid block).""" - pattern = r"```sql\s*(.*?)\s*```" - sql_blocks = re.findall(pattern, message, re.DOTALL) - for block in reversed(sql_blocks): - if len(block.strip()) > 6: - return block.strip() - return "" - - -def _extract_sql_generic_block(message: str) -> str: - """Extract SQL from generic ``` ... ``` blocks containing SELECT.""" - blocks = re.findall(r"```\s*(.*?)\s*```", message, re.DOTALL) - for block in reversed(blocks): - if "SELECT" in block.upper() and len(block.strip()) > 6: - return block.strip() - return "" - - -def _extract_sql_analyst(message: str) -> str: - """Extract SQL from ```json { "sql": "..." } ``` blocks.""" - block = re.search(r"```\s*json(.*?)```", message, re.DOTALL) - if block is None: - return "" - json_str = block.group(1) - idx_left = json_str.rfind("{") - idx_right = json_str.find("}") - if idx_left == -1 or idx_right == -1: - return "" - json_str = json_str[idx_left : idx_right + 1] - try: - return json.loads(json_str.replace("\\n", "\n").replace("\\'", "'"), strict=False).get("sql", "") - except Exception: - return "" - - -def _extract_sql_raw_select(message: str) -> str: - """Fallback: extract a raw SELECT statement.""" - match = re.search(r"(SELECT\s+.+?)(?:\n\n|$)", message, re.DOTALL | re.IGNORECASE) - if match: - return match.group(1).strip() - return "" - - -def extract_sql(response: str) -> str: - """Extract SQL from model response, following V6b's extraction pipeline. - - 1. Split on to isolate the answer portion - 2. Try ```sql blocks - 3. Try generic ``` blocks with SELECT - 4. Try ```json blocks with {"sql": ...} - 5. Fallback to raw SELECT statement - """ - if "" in response: - answer_part = response.split("", 1)[1] - else: - answer_part = response - - sql = _extract_sql_omnisql(answer_part) - if sql: - return sql - - sql = _extract_sql_generic_block(answer_part) - if sql: - return sql - - sql = _extract_sql_analyst(answer_part) - if sql: - return sql - - return _extract_sql_raw_select(answer_part) - - -# --------------------------------------------------------------------------- -# Format validation (mirrors V6b's validate_response_structure) -# --------------------------------------------------------------------------- - -def validate_response_format(response: str) -> bool: - """Check that the response has exactly one ... pair, properly nested.""" - start_positions = [m.start() for m in re.finditer(r"", response)] - end_positions = [m.start() for m in re.finditer(r"", response)] - - if len(start_positions) != 1 or len(end_positions) != 1: - return False - return start_positions[0] < end_positions[0] - - -# --------------------------------------------------------------------------- -# LIMIT addition (mirrors V6b's _add_limit_to_query) -# --------------------------------------------------------------------------- - -def _add_limit_to_query(query: str, limit: int = DEFAULT_LIMIT_NUMBER) -> str: - """Add LIMIT clause if the query doesn't already have one.""" - if not query: - return query - upper = query.upper() - if "LIMIT " in upper or "LIMIT\n" in upper or "LIMIT\t" in upper: - return query - return query.rstrip().rstrip(";").rstrip() + f" LIMIT {limit};" - - -# --------------------------------------------------------------------------- -# SQLite execution and comparison -# --------------------------------------------------------------------------- - -def _execute_sql(db_path: str, sql: str, timeout: float = SQL_TIMEOUT) -> frozenset | Exception: - """Execute SQL against a SQLite database and return result as frozenset of row tuples.""" - try: - conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5) - deadline = __import__("time").monotonic() + timeout - def _check_cancel(): - if __import__("time").monotonic() > deadline: - return 1 - return 0 - conn.set_progress_handler(_check_cancel, 1000) - cursor = conn.cursor() - cursor.execute(sql) - rows = cursor.fetchall() - conn.close() - return frozenset(rows) - except sqlite3.OperationalError as e: - if "interrupt" in str(e).lower(): - return TimeoutError(f"SQL execution exceeded {timeout}s") - return e - except Exception as e: - return e - - -def _execute_with_timeout(db_path: str, sql: str, timeout: float = SQL_TIMEOUT) -> frozenset | Exception: - """Execute SQL with a timeout using a thread pool. - - Uses shutdown(wait=False) to avoid blocking if the SQLite thread is stuck. - The SQLite progress handler provides cooperative cancellation. - """ - executor = ThreadPoolExecutor(max_workers=1) - future = executor.submit(_execute_sql, db_path, sql, timeout) - try: - return future.result(timeout=timeout + 2) - except FuturesTimeoutError: - return TimeoutError(f"SQL execution exceeded {timeout}s") - except Exception as e: - return e - finally: - executor.shutdown(wait=False) - - -def _compare_results( - db_path: str, - pred_sql: str, - gold_sqls: list[str], - timeout: float = SQL_TIMEOUT, -) -> tuple[float, bool]: - """Execute and compare predicted SQL against all gold SQLs. - - Returns (reward, execution_success) matching V6b's non-semantic-model logic: - - 1.0 if pred result == any gold result (frozenset match) - - 0.1 if pred executes but doesn't match any gold - - 0.0 if pred fails to execute - - Caches gold results within this call to avoid re-execution. - """ - pred_sql_limited = _add_limit_to_query(pred_sql) - pred_result = _execute_with_timeout(db_path, pred_sql_limited, timeout) - - if isinstance(pred_result, Exception): - return 0.0, False - - gold_cache: dict[str, frozenset | Exception] = {} - scores = [] - for gold_sql in gold_sqls: - if gold_sql not in gold_cache: - gold_sql_limited = _add_limit_to_query(gold_sql) - gold_cache[gold_sql] = _execute_with_timeout(db_path, gold_sql_limited, timeout) - gold_result = gold_cache[gold_sql] - - if isinstance(gold_result, Exception): - scores.append(0.0) - continue - - if pred_result == gold_result: - scores.append(1.0) - else: - scores.append(0.1) - - return (max(scores) if scores else 0.0), True - - -# --------------------------------------------------------------------------- -# Main entry point -# --------------------------------------------------------------------------- - -def compute_score(data_source, solution_str, ground_truth, extra_info=None, **kwargs): - """Compute reward score for SQL generation (verl custom_reward_function interface). - - Mirrors SnowflakeDialectSQLRewardManagerV6b logic with SQLite execution: - 1. Extract SQL from model response - 2. Validate response format ( tags) - 3. Execute predicted and gold SQL against SQLite - 4. Compare results (frozenset match) - 5. Apply format bonus - - Args: - data_source: Dataset identifier (e.g. "bird") - solution_str: Full model response text (decoded) - ground_truth: Gold SQL query string - extra_info: Dict with at minimum {"db_path": "/path/to/db.sqlite"}. - Optionally {"alternative_answers": [...]} for multiple gold SQLs. - - Returns: - dict with "score" (float), "format_correct" (float), "execution_success" (float) - """ - if extra_info is None: - return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} - - db_path = extra_info.get("db_path", "") - if not db_path: - return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} - - pred_sql = extract_sql(solution_str) - format_correct = float(bool(pred_sql) and validate_response_format(solution_str)) - - if not pred_sql: - return {"score": 0.0, "format_correct": 0.0, "execution_success": 0.0} - - alternative_answers = extra_info.get("alternative_answers") - if alternative_answers and len(alternative_answers) > 0: - gold_sqls = [str(s).strip() for s in alternative_answers if s and str(s).strip()] - else: - gold_sqls = [ground_truth] if ground_truth else [] - - if not gold_sqls: - return {"score": 0.0, "format_correct": format_correct, "execution_success": 0.0} - - reward, execution_success = _compare_results(db_path, pred_sql, gold_sqls) - - if format_correct: - reward = max(reward, FORMAT_REWARD_BONUS) - - return { - "score": reward, - "format_correct": format_correct, - "execution_success": float(execution_success), - } diff --git a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh b/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh deleted file mode 100755 index 8c17d88740d..00000000000 --- a/examples/arctic_rl/run_bird_grpo_arl_zorro_yes.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -# GRPO training for Qwen3-1.7B on BIRD SQL dataset -# Adapted from exp64_lr5e-6_416k (SnowflakeDialectSQLRewardManagerV6b config) -# -# 1 node, 8 GPUs -# -# Prerequisites: -# 1. Preprocess data: python examples/bird_sql/preprocess_bird.py -# 2. pip install func_timeout - -set -x - -SCRIPT_DIR="$(cd "$(dirname "$BASH_SOURCE[0]")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" - -export PYTHONPATH="$REPO_ROOT:$PYTHONPATH" - -export PYTHONUNBUFFERED=1 -export HYDRA_FULL_ERROR=1 -export RAY_DEDUP_LOGS=0 -export HF_HUB_OFFLINE=1 -export HF_HOME=/checkpoint/huggingface -export USE_ARCTIC_TRAINING_CLIENT=1 # entry point into ArcticRL -export CUDA_VISIBLE_DEVICES= - -USE_LEGACY_WORKER_IMPL=disable -ROLLOUT_NAME=arctic -NGPU_PER_NODE=1 - -USE_ARCTIC_RL=True -USE_ARCTIC_ZORRO=True -COLOCATE=False - -# BSZ=32 -# ROLL_N=16 -BSZ=4 -ROLL_N=4 -MAX_STEPS=10 -PROMPT_LEN=16384 -RESPONSE_LEN=4096 - -LOGGER=console -# LOGGER="['console','wandb']" - -#MODEL_SHORT=Qwen3-1.7B -MODEL_SHORT=Qwen3-0.6B - -MODEL=Qwen/$MODEL_SHORT - -experiment_name="bird_grpo_${MODEL_SHORT}_ngpu${NGPU_PER_NODE}_gbs${BSZ}_rolln${ROLL_N}_arl_zorro_yes" - -gpu_name=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader -i 0) -if [[ $gpu_name == *"H200"* ]]; then - echo "Running on Hopper" - flash_attention_v=flash_attention_3 -elif [[ $gpu_name == *"B200"* ]] || [[ $gpu_name == *"B300"* ]] ; then - echo "Running on Blackwell" - flash_attention_v=flash_attention_2 -else - echo "Running on unknown: $gpu_name; don't know which FA version to use" -fi - -# DATA_DIR="/data/snowflakesql/xyu/open-source-text2sql" -# TRAIN_FILES="$DATA_DIR/train.parquet" -# VAL_FILES="$DATA_DIR/val.parquet" - - -DATA_DIR="/code/shared/open-source-text2sql" -#TRAIN_FILES="$DATA_DIR/train.parquet" -#TRAIN_FILES="$DATA_DIR/train-1000.parquet" -TRAIN_FILES="$DATA_DIR/train-100.parquet" -VAL_FILES="$DATA_DIR/val.parquet" - -# LOG_PROBS=True -LOG_PROBS=False - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - algorithm.norm_adv_by_std_in_grpo=True \ - algorithm.use_kl_in_reward=False \ - algorithm.kl_ctrl.kl_coef=0.001 \ - data.train_files=$TRAIN_FILES \ - data.val_files=$VAL_FILES \ - data.train_batch_size=$BSZ \ - data.max_prompt_length=$PROMPT_LEN \ - data.max_response_length=$RESPONSE_LEN \ - data.filter_overlong_prompts=True \ - data.filter_overlong_prompts_workers=1 \ - data.truncation=left \ - actor_rollout_ref.model.path=$MODEL \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - +actor_rollout_ref.model.override_config.attn_implementation=$flash_attention_v \ - actor_rollout_ref.model.use_liger=True \ - actor_rollout_ref.actor.strategy=fsdp2 \ - actor_rollout_ref.actor.use_torch_compile=True \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.actor.kl_loss_coef=0.0 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.optim.lr=5e-6 \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.05 \ - actor_rollout_ref.actor.optim.betas='[0.9,0.95]' \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.fsdp_config.forward_prefetch=True \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=$ROLLOUT_NAME \ - actor_rollout_ref.rollout.agent.num_workers=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=$ROLL_N \ - actor_rollout_ref.rollout.temperature=1.0 \ - actor_rollout_ref.rollout.top_p=1.0 \ - actor_rollout_ref.rollout.calculate_log_probs=$LOG_PROBS \ - actor_rollout_ref.rollout.enforce_eager=False \ - actor_rollout_ref.rollout.max_num_seqs=256 \ - actor_rollout_ref.rollout.max_num_batched_tokens=32768 \ - actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ - actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ - actor_rollout_ref.rollout.val_kwargs.temperature=0 \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.rollout.val_kwargs.do_sample=False \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - actor_rollout_ref.nccl_timeout=1800 \ - trainer.use_legacy_worker_impl=$USE_LEGACY_WORKER_IMPL \ - trainer.use_arctic_rl=$USE_ARCTIC_RL \ - trainer.balance_batch=False \ - trainer.default_local_dir=/data-fast/sql-rl/$experiment_name \ - trainer.logger=$LOGGER \ - trainer.project_name=arctic_rl_bird_sql \ - trainer.experiment_name=$experiment_name \ - trainer.n_gpus_per_node=$NGPU_PER_NODE \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_epochs=10 \ - trainer.val_before_train=False \ - custom_reward_function.path="$SCRIPT_DIR/bird_reward.py" \ - custom_reward_function.name=compute_score \ - trainer.total_training_steps=$MAX_STEPS \ - arctic_rl.colocate=$COLOCATE \ - arctic_rl.training_gpus=2\ - arctic_rl.sampling_gpus=4\ - arctic_rl.log_prob_gpus=2\ - arctic_rl.use_zorro=$USE_ARCTIC_ZORRO \ - "$@" 2>&1 | tee $experiment_name.log From 4f11e10fe9c7ca5f7b438f2a2a18880e673103ef Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Tue, 28 Apr 2026 20:23:44 +0000 Subject: [PATCH 56/58] Add example --- .../arctic_rl/run_gsm8k_grpo_arl_zorro_yes.sh | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100755 examples/arctic_rl/run_gsm8k_grpo_arl_zorro_yes.sh diff --git a/examples/arctic_rl/run_gsm8k_grpo_arl_zorro_yes.sh b/examples/arctic_rl/run_gsm8k_grpo_arl_zorro_yes.sh new file mode 100755 index 00000000000..65b2aee2fce --- /dev/null +++ b/examples/arctic_rl/run_gsm8k_grpo_arl_zorro_yes.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 +export HYDRA_FULL_ERROR=1 +export RAY_DEDUP_LOGS=0 +export HF_HUB_OFFLINE=1 +export HF_HOME=/checkpoint/huggingface +# we want to make sure this runs on non-gpu client +export CUDA_VISIBLE_DEVICES= + +BSZ=1024 +UBS=32 +ROLL_N=5 +PROMPT_LENGTH=512 +RESPONSE_LENGTH=1024 +MAX_STEPS=100 + + +BSZ=8 +UBS=2 +ROLL_N=2 +PROMPT_LENGTH=512 +RESPONSE_LENGTH=1024 +MAX_STEPS=4 + + +experiment_name="qwen3-0.6B_arctic_gsm8k_grpo" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=/code/shared/gsm8k/train.parquet \ + data.val_files=/code/shared/gsm8k/test.parquet \ + data.train_batch_size=$BSZ \ + data.max_prompt_length=512 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.shuffle=False \ + +data.seed=42 \ + actor_rollout_ref.actor.data_loader_seed=42 \ + reward.num_workers=1 \ + actor_rollout_ref.rollout.agent.num_workers=4 \ + actor_rollout_ref.model.path=Qwen/Qwen3-0.6B \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=False \ + actor_rollout_ref.actor.ppo_mini_batch_size=$BSZ \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=$UBS \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + +actor_rollout_ref.model.override_config.attn_implementation=flash_attention_3 \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=$UBS \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=arctic \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.enforce_eager=True \ + actor_rollout_ref.rollout.n=$ROLL_N \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=$UBS \ + actor_rollout_ref.ref.fsdp_config.param_offload=False \ + actor_rollout_ref.ref.strategy=fsdp2 \ + algorithm.use_kl_in_reward=False \ + trainer.use_legacy_worker_impl=disable \ + trainer.use_arctic_rl=True \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.experiment_name=$experiment_name \ + trainer.project_name=arctic_gsm8k_grpo \ + trainer.val_before_train=False \ + trainer.n_gpus_per_node=1 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=-1 \ + trainer.total_epochs=15 \ + trainer.total_training_steps=${MAX_STEPS} \ + arctic_rl.colocate=False \ + arctic_rl.training_gpus=2\ + arctic_rl.sampling_gpus=2\ + arctic_rl.log_prob_gpus=0\ + arctic_rl.use_zorro=True \ + "$@" 2>&1 | tee $experiment_name.log From 0f9e1c6371afac9cd2faec1f56b9f8b39c0b6bbf Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Tue, 28 Apr 2026 20:38:27 +0000 Subject: [PATCH 57/58] Restore CI --- .github/{ => workflows}/workflows_old/README.md | 0 .github/{ => workflows}/workflows_old/check-pr-title.yml | 0 .github/{ => workflows}/workflows_old/cpu_unit_tests.yml | 0 .github/{ => workflows}/workflows_old/doc.yml | 0 .github/{ => workflows}/workflows_old/docker-build-ascend-a2.yml | 0 .github/{ => workflows}/workflows_old/docker-build-ascend-a3.yml | 0 .github/{ => workflows}/workflows_old/e2e_ascend.yml | 0 .github/{ => workflows}/workflows_old/e2e_fully_async_policy.yml | 0 .github/{ => workflows}/workflows_old/e2e_one_step_off_policy.yml | 0 .../workflows_old/e2e_one_step_off_policy_ascend.yml | 0 .../{ => workflows}/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml | 0 .github/{ => workflows}/workflows_old/e2e_ppo_trainer.yml | 0 .../workflows_old/e2e_ppo_trainer_megatron_sglang.yml | 0 .../workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml | 0 .../workflows_old/e2e_ppo_trainer_megatron_vllm.yml | 0 .../workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml | 0 .../workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml | 0 .../{ => workflows}/workflows_old/e2e_ppo_trainer_veomni_vllm.yml | 0 .github/{ => workflows}/workflows_old/e2e_sft_llm.yml | 0 .github/{ => workflows}/workflows_old/e2e_sft_llm_ascend.yml | 0 .github/{ => workflows}/workflows_old/e2e_sft_vlm.yml | 0 .github/{ => workflows}/workflows_old/gpu_unit_tests.yml | 0 .github/{ => workflows}/workflows_old/model.yml | 0 .github/{ => workflows}/workflows_old/model_ascend.yml | 0 .github/{ => workflows}/workflows_old/nightly_ascend.yml | 0 .github/{ => workflows}/workflows_old/npu_unit_tests.yml | 0 .github/{ => workflows}/workflows_old/precommit-autofix.yml | 0 .github/{ => workflows}/workflows_old/reward_model_sglang.yml | 0 .github/{ => workflows}/workflows_old/reward_model_vllm.yml | 0 .../{ => workflows}/workflows_old/reward_model_vllm_ascend.yml | 0 .github/{ => workflows}/workflows_old/sanity.yml | 0 .github/{ => workflows}/workflows_old/scorecard.yml | 0 .github/{ => workflows}/workflows_old/secrets_scan.yml | 0 .github/{ => workflows}/workflows_old/sgl.yml | 0 .github/{ => workflows}/workflows_old/type-coverage-check.yml | 0 .github/{ => workflows}/workflows_old/vllm.yml | 0 36 files changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/workflows_old/README.md (100%) rename .github/{ => workflows}/workflows_old/check-pr-title.yml (100%) rename .github/{ => workflows}/workflows_old/cpu_unit_tests.yml (100%) rename .github/{ => workflows}/workflows_old/doc.yml (100%) rename .github/{ => workflows}/workflows_old/docker-build-ascend-a2.yml (100%) rename .github/{ => workflows}/workflows_old/docker-build-ascend-a3.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_ascend.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_fully_async_policy.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_one_step_off_policy.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_one_step_off_policy_ascend.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_ppo_trainer.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_ppo_trainer_megatron_sglang.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_ppo_trainer_megatron_vllm.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_ppo_trainer_veomni_vllm.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_sft_llm.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_sft_llm_ascend.yml (100%) rename .github/{ => workflows}/workflows_old/e2e_sft_vlm.yml (100%) rename .github/{ => workflows}/workflows_old/gpu_unit_tests.yml (100%) rename .github/{ => workflows}/workflows_old/model.yml (100%) rename .github/{ => workflows}/workflows_old/model_ascend.yml (100%) rename .github/{ => workflows}/workflows_old/nightly_ascend.yml (100%) rename .github/{ => workflows}/workflows_old/npu_unit_tests.yml (100%) rename .github/{ => workflows}/workflows_old/precommit-autofix.yml (100%) rename .github/{ => workflows}/workflows_old/reward_model_sglang.yml (100%) rename .github/{ => workflows}/workflows_old/reward_model_vllm.yml (100%) rename .github/{ => workflows}/workflows_old/reward_model_vllm_ascend.yml (100%) rename .github/{ => workflows}/workflows_old/sanity.yml (100%) rename .github/{ => workflows}/workflows_old/scorecard.yml (100%) rename .github/{ => workflows}/workflows_old/secrets_scan.yml (100%) rename .github/{ => workflows}/workflows_old/sgl.yml (100%) rename .github/{ => workflows}/workflows_old/type-coverage-check.yml (100%) rename .github/{ => workflows}/workflows_old/vllm.yml (100%) diff --git a/.github/workflows_old/README.md b/.github/workflows/workflows_old/README.md similarity index 100% rename from .github/workflows_old/README.md rename to .github/workflows/workflows_old/README.md diff --git a/.github/workflows_old/check-pr-title.yml b/.github/workflows/workflows_old/check-pr-title.yml similarity index 100% rename from .github/workflows_old/check-pr-title.yml rename to .github/workflows/workflows_old/check-pr-title.yml diff --git a/.github/workflows_old/cpu_unit_tests.yml b/.github/workflows/workflows_old/cpu_unit_tests.yml similarity index 100% rename from .github/workflows_old/cpu_unit_tests.yml rename to .github/workflows/workflows_old/cpu_unit_tests.yml diff --git a/.github/workflows_old/doc.yml b/.github/workflows/workflows_old/doc.yml similarity index 100% rename from .github/workflows_old/doc.yml rename to .github/workflows/workflows_old/doc.yml diff --git a/.github/workflows_old/docker-build-ascend-a2.yml b/.github/workflows/workflows_old/docker-build-ascend-a2.yml similarity index 100% rename from .github/workflows_old/docker-build-ascend-a2.yml rename to .github/workflows/workflows_old/docker-build-ascend-a2.yml diff --git a/.github/workflows_old/docker-build-ascend-a3.yml b/.github/workflows/workflows_old/docker-build-ascend-a3.yml similarity index 100% rename from .github/workflows_old/docker-build-ascend-a3.yml rename to .github/workflows/workflows_old/docker-build-ascend-a3.yml diff --git a/.github/workflows_old/e2e_ascend.yml b/.github/workflows/workflows_old/e2e_ascend.yml similarity index 100% rename from .github/workflows_old/e2e_ascend.yml rename to .github/workflows/workflows_old/e2e_ascend.yml diff --git a/.github/workflows_old/e2e_fully_async_policy.yml b/.github/workflows/workflows_old/e2e_fully_async_policy.yml similarity index 100% rename from .github/workflows_old/e2e_fully_async_policy.yml rename to .github/workflows/workflows_old/e2e_fully_async_policy.yml diff --git a/.github/workflows_old/e2e_one_step_off_policy.yml b/.github/workflows/workflows_old/e2e_one_step_off_policy.yml similarity index 100% rename from .github/workflows_old/e2e_one_step_off_policy.yml rename to .github/workflows/workflows_old/e2e_one_step_off_policy.yml diff --git a/.github/workflows_old/e2e_one_step_off_policy_ascend.yml b/.github/workflows/workflows_old/e2e_one_step_off_policy_ascend.yml similarity index 100% rename from .github/workflows_old/e2e_one_step_off_policy_ascend.yml rename to .github/workflows/workflows_old/e2e_one_step_off_policy_ascend.yml diff --git a/.github/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml b/.github/workflows/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml similarity index 100% rename from .github/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml rename to .github/workflows/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml diff --git a/.github/workflows_old/e2e_ppo_trainer.yml b/.github/workflows/workflows_old/e2e_ppo_trainer.yml similarity index 100% rename from .github/workflows_old/e2e_ppo_trainer.yml rename to .github/workflows/workflows_old/e2e_ppo_trainer.yml diff --git a/.github/workflows_old/e2e_ppo_trainer_megatron_sglang.yml b/.github/workflows/workflows_old/e2e_ppo_trainer_megatron_sglang.yml similarity index 100% rename from .github/workflows_old/e2e_ppo_trainer_megatron_sglang.yml rename to .github/workflows/workflows_old/e2e_ppo_trainer_megatron_sglang.yml diff --git a/.github/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml b/.github/workflows/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml similarity index 100% rename from .github/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml rename to .github/workflows/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml diff --git a/.github/workflows_old/e2e_ppo_trainer_megatron_vllm.yml b/.github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm.yml similarity index 100% rename from .github/workflows_old/e2e_ppo_trainer_megatron_vllm.yml rename to .github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm.yml diff --git a/.github/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml b/.github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml similarity index 100% rename from .github/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml rename to .github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml diff --git a/.github/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml b/.github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml similarity index 100% rename from .github/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml rename to .github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml diff --git a/.github/workflows_old/e2e_ppo_trainer_veomni_vllm.yml b/.github/workflows/workflows_old/e2e_ppo_trainer_veomni_vllm.yml similarity index 100% rename from .github/workflows_old/e2e_ppo_trainer_veomni_vllm.yml rename to .github/workflows/workflows_old/e2e_ppo_trainer_veomni_vllm.yml diff --git a/.github/workflows_old/e2e_sft_llm.yml b/.github/workflows/workflows_old/e2e_sft_llm.yml similarity index 100% rename from .github/workflows_old/e2e_sft_llm.yml rename to .github/workflows/workflows_old/e2e_sft_llm.yml diff --git a/.github/workflows_old/e2e_sft_llm_ascend.yml b/.github/workflows/workflows_old/e2e_sft_llm_ascend.yml similarity index 100% rename from .github/workflows_old/e2e_sft_llm_ascend.yml rename to .github/workflows/workflows_old/e2e_sft_llm_ascend.yml diff --git a/.github/workflows_old/e2e_sft_vlm.yml b/.github/workflows/workflows_old/e2e_sft_vlm.yml similarity index 100% rename from .github/workflows_old/e2e_sft_vlm.yml rename to .github/workflows/workflows_old/e2e_sft_vlm.yml diff --git a/.github/workflows_old/gpu_unit_tests.yml b/.github/workflows/workflows_old/gpu_unit_tests.yml similarity index 100% rename from .github/workflows_old/gpu_unit_tests.yml rename to .github/workflows/workflows_old/gpu_unit_tests.yml diff --git a/.github/workflows_old/model.yml b/.github/workflows/workflows_old/model.yml similarity index 100% rename from .github/workflows_old/model.yml rename to .github/workflows/workflows_old/model.yml diff --git a/.github/workflows_old/model_ascend.yml b/.github/workflows/workflows_old/model_ascend.yml similarity index 100% rename from .github/workflows_old/model_ascend.yml rename to .github/workflows/workflows_old/model_ascend.yml diff --git a/.github/workflows_old/nightly_ascend.yml b/.github/workflows/workflows_old/nightly_ascend.yml similarity index 100% rename from .github/workflows_old/nightly_ascend.yml rename to .github/workflows/workflows_old/nightly_ascend.yml diff --git a/.github/workflows_old/npu_unit_tests.yml b/.github/workflows/workflows_old/npu_unit_tests.yml similarity index 100% rename from .github/workflows_old/npu_unit_tests.yml rename to .github/workflows/workflows_old/npu_unit_tests.yml diff --git a/.github/workflows_old/precommit-autofix.yml b/.github/workflows/workflows_old/precommit-autofix.yml similarity index 100% rename from .github/workflows_old/precommit-autofix.yml rename to .github/workflows/workflows_old/precommit-autofix.yml diff --git a/.github/workflows_old/reward_model_sglang.yml b/.github/workflows/workflows_old/reward_model_sglang.yml similarity index 100% rename from .github/workflows_old/reward_model_sglang.yml rename to .github/workflows/workflows_old/reward_model_sglang.yml diff --git a/.github/workflows_old/reward_model_vllm.yml b/.github/workflows/workflows_old/reward_model_vllm.yml similarity index 100% rename from .github/workflows_old/reward_model_vllm.yml rename to .github/workflows/workflows_old/reward_model_vllm.yml diff --git a/.github/workflows_old/reward_model_vllm_ascend.yml b/.github/workflows/workflows_old/reward_model_vllm_ascend.yml similarity index 100% rename from .github/workflows_old/reward_model_vllm_ascend.yml rename to .github/workflows/workflows_old/reward_model_vllm_ascend.yml diff --git a/.github/workflows_old/sanity.yml b/.github/workflows/workflows_old/sanity.yml similarity index 100% rename from .github/workflows_old/sanity.yml rename to .github/workflows/workflows_old/sanity.yml diff --git a/.github/workflows_old/scorecard.yml b/.github/workflows/workflows_old/scorecard.yml similarity index 100% rename from .github/workflows_old/scorecard.yml rename to .github/workflows/workflows_old/scorecard.yml diff --git a/.github/workflows_old/secrets_scan.yml b/.github/workflows/workflows_old/secrets_scan.yml similarity index 100% rename from .github/workflows_old/secrets_scan.yml rename to .github/workflows/workflows_old/secrets_scan.yml diff --git a/.github/workflows_old/sgl.yml b/.github/workflows/workflows_old/sgl.yml similarity index 100% rename from .github/workflows_old/sgl.yml rename to .github/workflows/workflows_old/sgl.yml diff --git a/.github/workflows_old/type-coverage-check.yml b/.github/workflows/workflows_old/type-coverage-check.yml similarity index 100% rename from .github/workflows_old/type-coverage-check.yml rename to .github/workflows/workflows_old/type-coverage-check.yml diff --git a/.github/workflows_old/vllm.yml b/.github/workflows/workflows_old/vllm.yml similarity index 100% rename from .github/workflows_old/vllm.yml rename to .github/workflows/workflows_old/vllm.yml From 20ca9ddac21986d1914a7e57b02854bfca54cf49 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Tue, 28 Apr 2026 20:45:18 +0000 Subject: [PATCH 58/58] Restore CI --- .github/workflows/{workflows_old => }/README.md | 0 .github/workflows/{workflows_old => }/check-pr-title.yml | 0 .github/workflows/{workflows_old => }/cpu_unit_tests.yml | 0 .github/workflows/{workflows_old => }/doc.yml | 0 .github/workflows/{workflows_old => }/docker-build-ascend-a2.yml | 0 .github/workflows/{workflows_old => }/docker-build-ascend-a3.yml | 0 .github/workflows/{workflows_old => }/e2e_ascend.yml | 0 .github/workflows/{workflows_old => }/e2e_fully_async_policy.yml | 0 .github/workflows/{workflows_old => }/e2e_one_step_off_policy.yml | 0 .../{workflows_old => }/e2e_one_step_off_policy_ascend.yml | 0 .../workflows/{workflows_old => }/e2e_ppo_grpo_trainer_trtllm.yml | 0 .github/workflows/{workflows_old => }/e2e_ppo_trainer.yml | 0 .../{workflows_old => }/e2e_ppo_trainer_megatron_sglang.yml | 0 .../{workflows_old => }/e2e_ppo_trainer_megatron_sglang_2.yml | 0 .../{workflows_old => }/e2e_ppo_trainer_megatron_vllm.yml | 0 .../{workflows_old => }/e2e_ppo_trainer_megatron_vllm_2.yml | 0 .../e2e_ppo_trainer_megatron_vllm_2_ascend.yml | 0 .../workflows/{workflows_old => }/e2e_ppo_trainer_veomni_vllm.yml | 0 .github/workflows/{workflows_old => }/e2e_sft_llm.yml | 0 .github/workflows/{workflows_old => }/e2e_sft_llm_ascend.yml | 0 .github/workflows/{workflows_old => }/e2e_sft_vlm.yml | 0 .github/workflows/{workflows_old => }/gpu_unit_tests.yml | 0 .github/workflows/{workflows_old => }/model.yml | 0 .github/workflows/{workflows_old => }/model_ascend.yml | 0 .github/workflows/{workflows_old => }/nightly_ascend.yml | 0 .github/workflows/{workflows_old => }/npu_unit_tests.yml | 0 .github/workflows/{workflows_old => }/precommit-autofix.yml | 0 .github/workflows/{workflows_old => }/reward_model_sglang.yml | 0 .github/workflows/{workflows_old => }/reward_model_vllm.yml | 0 .../workflows/{workflows_old => }/reward_model_vllm_ascend.yml | 0 .github/workflows/{workflows_old => }/sanity.yml | 0 .github/workflows/{workflows_old => }/scorecard.yml | 0 .github/workflows/{workflows_old => }/secrets_scan.yml | 0 .github/workflows/{workflows_old => }/sgl.yml | 0 .github/workflows/{workflows_old => }/type-coverage-check.yml | 0 .github/workflows/{workflows_old => }/vllm.yml | 0 36 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{workflows_old => }/README.md (100%) rename .github/workflows/{workflows_old => }/check-pr-title.yml (100%) rename .github/workflows/{workflows_old => }/cpu_unit_tests.yml (100%) rename .github/workflows/{workflows_old => }/doc.yml (100%) rename .github/workflows/{workflows_old => }/docker-build-ascend-a2.yml (100%) rename .github/workflows/{workflows_old => }/docker-build-ascend-a3.yml (100%) rename .github/workflows/{workflows_old => }/e2e_ascend.yml (100%) rename .github/workflows/{workflows_old => }/e2e_fully_async_policy.yml (100%) rename .github/workflows/{workflows_old => }/e2e_one_step_off_policy.yml (100%) rename .github/workflows/{workflows_old => }/e2e_one_step_off_policy_ascend.yml (100%) rename .github/workflows/{workflows_old => }/e2e_ppo_grpo_trainer_trtllm.yml (100%) rename .github/workflows/{workflows_old => }/e2e_ppo_trainer.yml (100%) rename .github/workflows/{workflows_old => }/e2e_ppo_trainer_megatron_sglang.yml (100%) rename .github/workflows/{workflows_old => }/e2e_ppo_trainer_megatron_sglang_2.yml (100%) rename .github/workflows/{workflows_old => }/e2e_ppo_trainer_megatron_vllm.yml (100%) rename .github/workflows/{workflows_old => }/e2e_ppo_trainer_megatron_vllm_2.yml (100%) rename .github/workflows/{workflows_old => }/e2e_ppo_trainer_megatron_vllm_2_ascend.yml (100%) rename .github/workflows/{workflows_old => }/e2e_ppo_trainer_veomni_vllm.yml (100%) rename .github/workflows/{workflows_old => }/e2e_sft_llm.yml (100%) rename .github/workflows/{workflows_old => }/e2e_sft_llm_ascend.yml (100%) rename .github/workflows/{workflows_old => }/e2e_sft_vlm.yml (100%) rename .github/workflows/{workflows_old => }/gpu_unit_tests.yml (100%) rename .github/workflows/{workflows_old => }/model.yml (100%) rename .github/workflows/{workflows_old => }/model_ascend.yml (100%) rename .github/workflows/{workflows_old => }/nightly_ascend.yml (100%) rename .github/workflows/{workflows_old => }/npu_unit_tests.yml (100%) rename .github/workflows/{workflows_old => }/precommit-autofix.yml (100%) rename .github/workflows/{workflows_old => }/reward_model_sglang.yml (100%) rename .github/workflows/{workflows_old => }/reward_model_vllm.yml (100%) rename .github/workflows/{workflows_old => }/reward_model_vllm_ascend.yml (100%) rename .github/workflows/{workflows_old => }/sanity.yml (100%) rename .github/workflows/{workflows_old => }/scorecard.yml (100%) rename .github/workflows/{workflows_old => }/secrets_scan.yml (100%) rename .github/workflows/{workflows_old => }/sgl.yml (100%) rename .github/workflows/{workflows_old => }/type-coverage-check.yml (100%) rename .github/workflows/{workflows_old => }/vllm.yml (100%) diff --git a/.github/workflows/workflows_old/README.md b/.github/workflows/README.md similarity index 100% rename from .github/workflows/workflows_old/README.md rename to .github/workflows/README.md diff --git a/.github/workflows/workflows_old/check-pr-title.yml b/.github/workflows/check-pr-title.yml similarity index 100% rename from .github/workflows/workflows_old/check-pr-title.yml rename to .github/workflows/check-pr-title.yml diff --git a/.github/workflows/workflows_old/cpu_unit_tests.yml b/.github/workflows/cpu_unit_tests.yml similarity index 100% rename from .github/workflows/workflows_old/cpu_unit_tests.yml rename to .github/workflows/cpu_unit_tests.yml diff --git a/.github/workflows/workflows_old/doc.yml b/.github/workflows/doc.yml similarity index 100% rename from .github/workflows/workflows_old/doc.yml rename to .github/workflows/doc.yml diff --git a/.github/workflows/workflows_old/docker-build-ascend-a2.yml b/.github/workflows/docker-build-ascend-a2.yml similarity index 100% rename from .github/workflows/workflows_old/docker-build-ascend-a2.yml rename to .github/workflows/docker-build-ascend-a2.yml diff --git a/.github/workflows/workflows_old/docker-build-ascend-a3.yml b/.github/workflows/docker-build-ascend-a3.yml similarity index 100% rename from .github/workflows/workflows_old/docker-build-ascend-a3.yml rename to .github/workflows/docker-build-ascend-a3.yml diff --git a/.github/workflows/workflows_old/e2e_ascend.yml b/.github/workflows/e2e_ascend.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_ascend.yml rename to .github/workflows/e2e_ascend.yml diff --git a/.github/workflows/workflows_old/e2e_fully_async_policy.yml b/.github/workflows/e2e_fully_async_policy.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_fully_async_policy.yml rename to .github/workflows/e2e_fully_async_policy.yml diff --git a/.github/workflows/workflows_old/e2e_one_step_off_policy.yml b/.github/workflows/e2e_one_step_off_policy.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_one_step_off_policy.yml rename to .github/workflows/e2e_one_step_off_policy.yml diff --git a/.github/workflows/workflows_old/e2e_one_step_off_policy_ascend.yml b/.github/workflows/e2e_one_step_off_policy_ascend.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_one_step_off_policy_ascend.yml rename to .github/workflows/e2e_one_step_off_policy_ascend.yml diff --git a/.github/workflows/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml b/.github/workflows/e2e_ppo_grpo_trainer_trtllm.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_ppo_grpo_trainer_trtllm.yml rename to .github/workflows/e2e_ppo_grpo_trainer_trtllm.yml diff --git a/.github/workflows/workflows_old/e2e_ppo_trainer.yml b/.github/workflows/e2e_ppo_trainer.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_ppo_trainer.yml rename to .github/workflows/e2e_ppo_trainer.yml diff --git a/.github/workflows/workflows_old/e2e_ppo_trainer_megatron_sglang.yml b/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_ppo_trainer_megatron_sglang.yml rename to .github/workflows/e2e_ppo_trainer_megatron_sglang.yml diff --git a/.github/workflows/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml b/.github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_ppo_trainer_megatron_sglang_2.yml rename to .github/workflows/e2e_ppo_trainer_megatron_sglang_2.yml diff --git a/.github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm.yml b/.github/workflows/e2e_ppo_trainer_megatron_vllm.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm.yml rename to .github/workflows/e2e_ppo_trainer_megatron_vllm.yml diff --git a/.github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml b/.github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm_2.yml rename to .github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml diff --git a/.github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml b/.github/workflows/e2e_ppo_trainer_megatron_vllm_2_ascend.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_ppo_trainer_megatron_vllm_2_ascend.yml rename to .github/workflows/e2e_ppo_trainer_megatron_vllm_2_ascend.yml diff --git a/.github/workflows/workflows_old/e2e_ppo_trainer_veomni_vllm.yml b/.github/workflows/e2e_ppo_trainer_veomni_vllm.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_ppo_trainer_veomni_vllm.yml rename to .github/workflows/e2e_ppo_trainer_veomni_vllm.yml diff --git a/.github/workflows/workflows_old/e2e_sft_llm.yml b/.github/workflows/e2e_sft_llm.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_sft_llm.yml rename to .github/workflows/e2e_sft_llm.yml diff --git a/.github/workflows/workflows_old/e2e_sft_llm_ascend.yml b/.github/workflows/e2e_sft_llm_ascend.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_sft_llm_ascend.yml rename to .github/workflows/e2e_sft_llm_ascend.yml diff --git a/.github/workflows/workflows_old/e2e_sft_vlm.yml b/.github/workflows/e2e_sft_vlm.yml similarity index 100% rename from .github/workflows/workflows_old/e2e_sft_vlm.yml rename to .github/workflows/e2e_sft_vlm.yml diff --git a/.github/workflows/workflows_old/gpu_unit_tests.yml b/.github/workflows/gpu_unit_tests.yml similarity index 100% rename from .github/workflows/workflows_old/gpu_unit_tests.yml rename to .github/workflows/gpu_unit_tests.yml diff --git a/.github/workflows/workflows_old/model.yml b/.github/workflows/model.yml similarity index 100% rename from .github/workflows/workflows_old/model.yml rename to .github/workflows/model.yml diff --git a/.github/workflows/workflows_old/model_ascend.yml b/.github/workflows/model_ascend.yml similarity index 100% rename from .github/workflows/workflows_old/model_ascend.yml rename to .github/workflows/model_ascend.yml diff --git a/.github/workflows/workflows_old/nightly_ascend.yml b/.github/workflows/nightly_ascend.yml similarity index 100% rename from .github/workflows/workflows_old/nightly_ascend.yml rename to .github/workflows/nightly_ascend.yml diff --git a/.github/workflows/workflows_old/npu_unit_tests.yml b/.github/workflows/npu_unit_tests.yml similarity index 100% rename from .github/workflows/workflows_old/npu_unit_tests.yml rename to .github/workflows/npu_unit_tests.yml diff --git a/.github/workflows/workflows_old/precommit-autofix.yml b/.github/workflows/precommit-autofix.yml similarity index 100% rename from .github/workflows/workflows_old/precommit-autofix.yml rename to .github/workflows/precommit-autofix.yml diff --git a/.github/workflows/workflows_old/reward_model_sglang.yml b/.github/workflows/reward_model_sglang.yml similarity index 100% rename from .github/workflows/workflows_old/reward_model_sglang.yml rename to .github/workflows/reward_model_sglang.yml diff --git a/.github/workflows/workflows_old/reward_model_vllm.yml b/.github/workflows/reward_model_vllm.yml similarity index 100% rename from .github/workflows/workflows_old/reward_model_vllm.yml rename to .github/workflows/reward_model_vllm.yml diff --git a/.github/workflows/workflows_old/reward_model_vllm_ascend.yml b/.github/workflows/reward_model_vllm_ascend.yml similarity index 100% rename from .github/workflows/workflows_old/reward_model_vllm_ascend.yml rename to .github/workflows/reward_model_vllm_ascend.yml diff --git a/.github/workflows/workflows_old/sanity.yml b/.github/workflows/sanity.yml similarity index 100% rename from .github/workflows/workflows_old/sanity.yml rename to .github/workflows/sanity.yml diff --git a/.github/workflows/workflows_old/scorecard.yml b/.github/workflows/scorecard.yml similarity index 100% rename from .github/workflows/workflows_old/scorecard.yml rename to .github/workflows/scorecard.yml diff --git a/.github/workflows/workflows_old/secrets_scan.yml b/.github/workflows/secrets_scan.yml similarity index 100% rename from .github/workflows/workflows_old/secrets_scan.yml rename to .github/workflows/secrets_scan.yml diff --git a/.github/workflows/workflows_old/sgl.yml b/.github/workflows/sgl.yml similarity index 100% rename from .github/workflows/workflows_old/sgl.yml rename to .github/workflows/sgl.yml diff --git a/.github/workflows/workflows_old/type-coverage-check.yml b/.github/workflows/type-coverage-check.yml similarity index 100% rename from .github/workflows/workflows_old/type-coverage-check.yml rename to .github/workflows/type-coverage-check.yml diff --git a/.github/workflows/workflows_old/vllm.yml b/.github/workflows/vllm.yml similarity index 100% rename from .github/workflows/workflows_old/vllm.yml rename to .github/workflows/vllm.yml