agent_cls: skyrl_agent.agents.oh_codeact.OHCodeActAgent
task: skyrl_agent.tasks.swebench.utils.SWEBenchTask
tools:
enable_bash_tool: true
enable_browsing: false
enable_llm_editor: false
enable_editor: true
enable_jupyter: false
enable_cmd: true
enable_think: false
enable_finish: true
enable_mcp: false
enable_search: false
data:
instance_key: instance
instance_id_key: instance.instance_id
data_source_key: data_source
generator:
infer_backend: verl
backend_config: null
num_trajectories: 8
max_iterations: 25 # 50
max_prompt_length: 16384
sampling_params:
temperature: 0.6
top_p: 0.95
top_k: 20
repetition_penalty: 1.5
max_tokens: 16384
val_config:
num_trajectories: 1
sampling_params:
temperature: 1
top_p: 0.95
max_tokens: 16384
remove_think_tokens: false
vision_is_active: false
qwen3_enable_thinking: false
qwen3_acc_thinking: false
dispatcher:
type: async_pipeline
max_parallel_agents: 8
max_eval_parallel_agents: 8
val_config:
max_parallel_agents: 4
max_eval_parallel_agents: 4
set -x
train_data="/mnt/moe-nas/public/datasets/SWE/SkyRL-v0-293-data/train_30_39.parquet"
test_data="/mnt/moe-nas/public/datasets/SWE/SkyRL-v0-293-data/validation.parquet"
export WANDB_API_KEY="9b2ccd555259e83bd122fa4d8c6d16e55b003942"
export CUDA_VISIBLE_DEVICES=0,1
export RUNTIME=local
export OPENHANDS_API_KEY="sandbox-remote"
export SANDBOX_API_KEY="sandbox-remote"
export ALLHANDS_API_KEY="sandbox-remote"
export SANDBOX_REMOTE_RUNTIME_API_URL="http://10.200.51.249:3000"
export EVAL_DOCKER_IMAGE_PREFIX="021harbor.zero2x.org/xingyaoww"
export EVAL_DOCKER_IMAGE_PREFIX="021harbor.zero2x.org/namanjain12"
export SANDBOX_USE_RUNTIME_VOLUME="true"
export RAY_ENABLE_UV_RUN_RUNTIME_ENV=0
export RAY_PYTHON_PATH="/mnt/moe-nas/home/zxn/code/SkyRL/skyrl-agent/.venv/bin/python"
MODEL=/mnt/moe-nas/public/models/Qwen3-8B
NNODES=1
SP_SIZE=1
TP_SIZE=2
NNODES=1
SP_SIZE=2
TP_SIZE=2
echo "=== .env 文件内容 ==="
cat /mnt/moe-nas/home/zxn/code/SkyRL/skyrl-agent/.env
echo "===================="
UV_HTTP_TIMEOUT=3000 RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 OPENHANDS_API_KEY="sandbox-remote" EVAL_DOCKER_IMAGE_PREFIX="021harbor.zero2x.org/xingyaoww" SANDBOX_USE_RUNTIME_VOLUME=True RUNTIME=remote uv run --active --extra verl --env-file /mnt/moe-nas/home/zxn/code/SkyRL/skyrl-agent/.env -i https://pypi.tuna.tsinghua.edu.cn/simple
-m skyrl_agent.integrations.verl.verl_main_ppo
algorithm.adv_estimator=loop
data.train_files=$train_data
data.val_files=$test_data
data.dataloader_num_workers=0
data.train_batch_size=4
data.max_prompt_length=8192
data.max_response_length=1024
data.filter_overlong_prompts=True
data.truncation='error'
data.return_raw_chat=true
actor_rollout_ref.model.path=$MODEL
actor_rollout_ref.actor.loss_agg_mode=seq-mean-token-sum
actor_rollout_ref.actor.optim.lr=1e-6
actor_rollout_ref.model.use_remove_padding=True
actor_rollout_ref.actor.ppo_mini_batch_size=4
actor_rollout_ref.actor.use_dynamic_bsz=False
actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1
actor_rollout_ref.actor.use_kl_loss=True
actor_rollout_ref.actor.kl_loss_coef=0.001
actor_rollout_ref.actor.kl_loss_type=low_var_kl
actor_rollout_ref.actor.entropy_coeff=0
actor_rollout_ref.actor.clip_ratio_high=0.28
actor_rollout_ref.model.enable_gradient_checkpointing=True
actor_rollout_ref.actor.fsdp_config.param_offload=True
actor_rollout_ref.actor.fsdp_config.optimizer_offload=True
actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=False
actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1
actor_rollout_ref.actor.ulysses_sequence_parallel_size=$SP_SIZE
actor_rollout_ref.rollout.tensor_model_parallel_size=$TP_SIZE
actor_rollout_ref.rollout.enforce_eager=False
actor_rollout_ref.rollout.free_cache_engine=True
actor_rollout_ref.rollout.name=vllm
actor_rollout_ref.rollout.mode=async
actor_rollout_ref.rollout.gpu_memory_utilization=0.7
actor_rollout_ref.rollout.n=2
actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1
actor_rollout_ref.ref.fsdp_config.param_offload=True
algorithm.use_kl_in_reward=False
algorithm.norm_adv_by_std_in_grpo=False
trainer.val_before_train=False
trainer.critic_warmup=0
trainer.logger='["console"]'
trainer.project_name='qwen3-4b-swebench'
trainer.experiment_name='verl-qwen3-4b-r24-0429-k8s'
trainer.n_gpus_per_node=8
trainer.nnodes=$NNODES
trainer.max_actor_ckpt_to_keep=10
trainer.save_freq=100
trainer.default_local_dir=/mnt/moe-nas/home/zxn/code/SkyRL/local_storage/ckpts/skyagent-32b-r2e-verl/skyagent-verl-30b-r2e-4500-loop-sep27
trainer.test_freq=2000
trainer.total_epochs=15
trainer.validation_data_dir=/mnt/moe-nas/home/zxn/code/SkyRL/local_storage/rollouts/verl_grpo_skyagent/qwen3_4b_function_rm_val
+skyrl_agent.task_yaml="./examples/run_verl/verl_oh.yaml"
+skyrl_agent.num_trajectories=8 \
# trainer.rollout_data_dir=/mnt/moe_nas/home/zxn/code/SkyRL/local_storage/rollouts/verl_grpo_skyagent/qwen3_4b_function_rm \
+trainer.remote_anyscale_upload=True \
+trainer.remote_upload_dir=remote_ckpts/skyagent-32b-r2e-verl/skyagent-verl-30b-r2e-4500-loop-sep27 $@
trainer.resume_from_path=/mnt/local_storage/ckpts/skyagent-32b-r2e-verl/skyagent-verl-32b-r2e-4500-loop-aug20/global_step_14 \
trainer.resume_mode=resume_path\
trainer.resume_from_path=/mnt/local_storage/ckpts/skyagent-32b-r2e-verl/skyagent-verl-32b-r2e-4500-acc-thinking-mini4-aug15/global_step_26 \
上面是实验的基本配置，拉起训练后，一直出现
Bad LLM Response: Function 'str_replace-editor' not found in available tools: ['execute_bash', 'finish', 'str_replace_editor']\n[Error occurred in processing last action]"}, {'role': 'user', 'content': "\n\nBad LLM Response: Function 'str_replaceEditor' not found in available tools: ['execute_bash', 'finish', 'str_replace_editor']\n[Error occurred in processing last action]"}, {'role': 'user', 'content': "\n\nBad LLM Response: Function 'str_replace Editor' not found in available tools: ['execute_bash', 'finish', 'str_replace_editor']\n[Error occurred in processing last action]"}, {'role': 'user', 'content': "\n\nBad LLM Response: Missing required parameters for function 'str_replace_editor': {'command', 'path'}\n[Error occurred in processing last action]"}, {'role': 'user', 'content': "\n\nBad LLM Response: Missing required parameters for function 'str_replace_editor': {'command', 'path'}\n[Error occurred in processing last action]\nSteps remaining: 20."}]
是否遇到这种问题呢，放在slime里是没有上述问题的。
waiting for reply
agent_cls: skyrl_agent.agents.oh_codeact.OHCodeActAgent
task: skyrl_agent.tasks.swebench.utils.SWEBenchTask
tools:
enable_bash_tool: true
enable_browsing: false
enable_llm_editor: false
enable_editor: true
enable_jupyter: false
enable_cmd: true
enable_think: false
enable_finish: true
enable_mcp: false
enable_search: false
data:
instance_key: instance
instance_id_key: instance.instance_id
data_source_key: data_source
generator:
infer_backend: verl
backend_config: null
num_trajectories: 8
max_iterations: 25 # 50
max_prompt_length: 16384
sampling_params:
temperature: 0.6
top_p: 0.95
top_k: 20
repetition_penalty: 1.5
max_tokens: 16384
val_config:
num_trajectories: 1
sampling_params:
temperature: 1
top_p: 0.95
max_tokens: 16384
remove_think_tokens: false
vision_is_active: false
qwen3_enable_thinking: false
qwen3_acc_thinking: false
dispatcher:
type: async_pipeline
max_parallel_agents: 8
max_eval_parallel_agents: 8
val_config:
max_parallel_agents: 4
max_eval_parallel_agents: 4
set -x
train_data="/mnt/moe-nas/public/datasets/SWE/SkyRL-v0-293-data/train_30_39.parquet"
test_data="/mnt/moe-nas/public/datasets/SWE/SkyRL-v0-293-data/validation.parquet"
export WANDB_API_KEY="9b2ccd555259e83bd122fa4d8c6d16e55b003942"
export CUDA_VISIBLE_DEVICES=0,1
export RUNTIME=local
export OPENHANDS_API_KEY="sandbox-remote"
export SANDBOX_API_KEY="sandbox-remote"
export ALLHANDS_API_KEY="sandbox-remote"
export SANDBOX_REMOTE_RUNTIME_API_URL="http://10.200.51.249:3000"
export EVAL_DOCKER_IMAGE_PREFIX="021harbor.zero2x.org/xingyaoww"
export EVAL_DOCKER_IMAGE_PREFIX="021harbor.zero2x.org/namanjain12"
export SANDBOX_USE_RUNTIME_VOLUME="true"
export RAY_ENABLE_UV_RUN_RUNTIME_ENV=0
export RAY_PYTHON_PATH="/mnt/moe-nas/home/zxn/code/SkyRL/skyrl-agent/.venv/bin/python"
MODEL=/mnt/moe-nas/public/models/Qwen3-8B
NNODES=1
SP_SIZE=1
TP_SIZE=2
NNODES=1
SP_SIZE=2
TP_SIZE=2
echo "=== .env 文件内容 ==="
cat /mnt/moe-nas/home/zxn/code/SkyRL/skyrl-agent/.env
echo "===================="
UV_HTTP_TIMEOUT=3000 RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 OPENHANDS_API_KEY="sandbox-remote" EVAL_DOCKER_IMAGE_PREFIX="021harbor.zero2x.org/xingyaoww" SANDBOX_USE_RUNTIME_VOLUME=True RUNTIME=remote uv run --active --extra verl --env-file /mnt/moe-nas/home/zxn/code/SkyRL/skyrl-agent/.env -i https://pypi.tuna.tsinghua.edu.cn/simple
-m skyrl_agent.integrations.verl.verl_main_ppo
algorithm.adv_estimator=loop
data.train_files=$train_data
data.val_files=$test_data
data.dataloader_num_workers=0
data.train_batch_size=4
data.max_prompt_length=8192
data.max_response_length=1024
data.filter_overlong_prompts=True
data.truncation='error'
data.return_raw_chat=true
actor_rollout_ref.model.path=$MODEL
actor_rollout_ref.actor.loss_agg_mode=seq-mean-token-sum
actor_rollout_ref.actor.optim.lr=1e-6
actor_rollout_ref.model.use_remove_padding=True
actor_rollout_ref.actor.ppo_mini_batch_size=4
actor_rollout_ref.actor.use_dynamic_bsz=False
actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1
actor_rollout_ref.actor.use_kl_loss=True
actor_rollout_ref.actor.kl_loss_coef=0.001
actor_rollout_ref.actor.kl_loss_type=low_var_kl
actor_rollout_ref.actor.entropy_coeff=0
actor_rollout_ref.actor.clip_ratio_high=0.28
actor_rollout_ref.model.enable_gradient_checkpointing=True
actor_rollout_ref.actor.fsdp_config.param_offload=True
actor_rollout_ref.actor.fsdp_config.optimizer_offload=True
actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=False
actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1
actor_rollout_ref.actor.ulysses_sequence_parallel_size=$SP_SIZE
actor_rollout_ref.rollout.tensor_model_parallel_size=$TP_SIZE
actor_rollout_ref.rollout.enforce_eager=False
actor_rollout_ref.rollout.free_cache_engine=True
actor_rollout_ref.rollout.name=vllm
actor_rollout_ref.rollout.mode=async
actor_rollout_ref.rollout.gpu_memory_utilization=0.7
actor_rollout_ref.rollout.n=2
actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1
actor_rollout_ref.ref.fsdp_config.param_offload=True
algorithm.use_kl_in_reward=False
algorithm.norm_adv_by_std_in_grpo=False
trainer.val_before_train=False
trainer.critic_warmup=0
trainer.logger='["console"]'
trainer.project_name='qwen3-4b-swebench'
trainer.experiment_name='verl-qwen3-4b-r24-0429-k8s'
trainer.n_gpus_per_node=8
trainer.nnodes=$NNODES
trainer.max_actor_ckpt_to_keep=10
trainer.save_freq=100
trainer.default_local_dir=/mnt/moe-nas/home/zxn/code/SkyRL/local_storage/ckpts/skyagent-32b-r2e-verl/skyagent-verl-30b-r2e-4500-loop-sep27
trainer.test_freq=2000
trainer.total_epochs=15
trainer.validation_data_dir=/mnt/moe-nas/home/zxn/code/SkyRL/local_storage/rollouts/verl_grpo_skyagent/qwen3_4b_function_rm_val
+skyrl_agent.task_yaml="./examples/run_verl/verl_oh.yaml"
+skyrl_agent.num_trajectories=8 \
+trainer.remote_anyscale_upload=True \
+trainer.remote_upload_dir=remote_ckpts/skyagent-32b-r2e-verl/skyagent-verl-30b-r2e-4500-loop-sep27 $@
trainer.resume_from_path=/mnt/local_storage/ckpts/skyagent-32b-r2e-verl/skyagent-verl-32b-r2e-4500-loop-aug20/global_step_14 \
trainer.resume_mode=resume_path\
trainer.resume_from_path=/mnt/local_storage/ckpts/skyagent-32b-r2e-verl/skyagent-verl-32b-r2e-4500-acc-thinking-mini4-aug15/global_step_26 \
上面是实验的基本配置，拉起训练后，一直出现
Bad LLM Response: Function 'str_replace-editor' not found in available tools: ['execute_bash', 'finish', 'str_replace_editor']\n[Error occurred in processing last action]"}, {'role': 'user', 'content': "\n\nBad LLM Response: Function 'str_replaceEditor' not found in available tools: ['execute_bash', 'finish', 'str_replace_editor']\n[Error occurred in processing last action]"}, {'role': 'user', 'content': "\n\nBad LLM Response: Function 'str_replace Editor' not found in available tools: ['execute_bash', 'finish', 'str_replace_editor']\n[Error occurred in processing last action]"}, {'role': 'user', 'content': "\n\nBad LLM Response: Missing required parameters for function 'str_replace_editor': {'command', 'path'}\n[Error occurred in processing last action]"}, {'role': 'user', 'content': "\n\nBad LLM Response: Missing required parameters for function 'str_replace_editor': {'command', 'path'}\n[Error occurred in processing last action]\nSteps remaining: 20."}]
是否遇到这种问题呢，放在slime里是没有上述问题的。
waiting for reply