update examples

Author: kylemontgomery1

examples/deepcoder/train_deepcoder.py

@@ -7,7 +7,7 @@
 from rllm.trainer.agent_trainer import AgentTrainer
 
 
-@hydra.main(config_path="pkg://rllm.trainer.config", config_name="ppo_trainer", version_base=None)
+@hydra.main(config_path="pkg://rllm.trainer.config", config_name="agent_ppo_trainer", version_base=None)
 def main(config):
     train_dataset = DatasetRegistry.load_dataset("deepcoder", "train")
     test_dataset = DatasetRegistry.load_dataset("deepcoder", "test")

examples/deepcoder/train_deepcoder_16k.sh

@@ -43,7 +43,6 @@ python3 -m examples.deepcoder.train_deepcoder \
     actor_rollout_ref.rollout.tensor_model_parallel_size=2 \
     actor_rollout_ref.rollout.name=vllm \
     actor_rollout_ref.rollout.mode="async" \
-    actor_rollout_ref.rollout.chat_scheduler=verl.schedulers.completions_scheduler.CompletionsScheduler \
     actor_rollout_ref.rollout.enforce_eager=False \
     actor_rollout_ref.rollout.temperature=0.6 \
     actor_rollout_ref.rollout.top_p=0.95 \
@@ -56,8 +55,7 @@ python3 -m examples.deepcoder.train_deepcoder \
     actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \
     actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
     algorithm.kl_ctrl.kl_coef=0.001 \
-    algorithm.mask_truncated_samples=True \
-    algorithm.clip_advantages=False \
+    rllm.mask_truncated_samples=True \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='rllm-deepcoder' \
@@ -68,6 +66,6 @@ python3 -m examples.deepcoder.train_deepcoder \
     trainer.save_freq=10 \
     trainer.test_freq=10 \
     trainer.default_hdfs_dir=null \
-    agent.max_steps=1 \
-    agent.use_stepwise_advantage=False \
+    rllm.agent.max_steps=1 \
+    rllm.stepwise_advantage.enable=False \
     trainer.total_epochs=100 

examples/deepcoder/train_deepcoder_32k.sh

@@ -43,7 +43,6 @@ python3 -m examples.deepcoder.train_deepcoder \
     actor_rollout_ref.rollout.tensor_model_parallel_size=2 \
     actor_rollout_ref.rollout.name=vllm \
     actor_rollout_ref.rollout.mode="async" \
-    actor_rollout_ref.rollout.chat_scheduler=verl.schedulers.completions_scheduler.CompletionsScheduler \
     actor_rollout_ref.rollout.enforce_eager=False \
     actor_rollout_ref.rollout.temperature=0.6 \
     actor_rollout_ref.rollout.top_p=0.95 \
@@ -56,8 +55,7 @@ python3 -m examples.deepcoder.train_deepcoder \
     actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \
     actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
     algorithm.kl_ctrl.kl_coef=0.001 \
-    algorithm.mask_truncated_samples=False \
-    algorithm.clip_advantages=False \
+    rllm.mask_truncated_samples=False \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='rllm-deepcoder' \
@@ -68,6 +66,6 @@ python3 -m examples.deepcoder.train_deepcoder \
     trainer.save_freq=10 \
     trainer.test_freq=10 \
     trainer.default_hdfs_dir=null \
-    agent.max_steps=1 \
-    agent.use_stepwise_advantage=False \
+    rllm.agent.max_steps=1 \
+    rllm.stepwise_advantage.enable=False \
     trainer.total_epochs=100 

examples/deepscaler/train_deepscaler.py

@@ -7,7 +7,7 @@
 from rllm.trainer.agent_trainer import AgentTrainer
 
 
-@hydra.main(config_path="pkg://rllm.trainer.config", config_name="ppo_trainer", version_base=None)
+@hydra.main(config_path="pkg://rllm.trainer.config", config_name="agent_ppo_trainer", version_base=None)
 def main(config):
     train_dataset = DatasetRegistry.load_dataset("deepscaler_math", "train")
     test_dataset = DatasetRegistry.load_dataset("aime2024", "test")

examples/deepscaler/train_deepscaler_16k.sh

@@ -30,14 +30,12 @@ python3 -m examples.deepscaler.train_deepscaler \
     actor_rollout_ref.actor.kl_loss_coef=0.001 \
     actor_rollout_ref.actor.kl_loss_type=low_var_kl \
     actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \
-    actor_rollout_ref.actor.grad_norm_threshold=10 \
     actor_rollout_ref.model.enable_gradient_checkpointing=True \
     actor_rollout_ref.actor.fsdp_config.param_offload=True \
     actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \
     actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
     actor_rollout_ref.rollout.name=vllm \
     actor_rollout_ref.rollout.mode="async" \
-    actor_rollout_ref.rollout.chat_scheduler=verl.schedulers.completions_scheduler.CompletionsScheduler \
     actor_rollout_ref.rollout.enforce_eager=False \
     actor_rollout_ref.rollout.temperature=0.6 \
     actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \
@@ -50,8 +48,7 @@ python3 -m examples.deepscaler.train_deepscaler \
     actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
     actor_rollout_ref.actor.entropy_coeff=0 \
     algorithm.kl_ctrl.kl_coef=0.001 \
-    algorithm.mask_truncated_samples=False \
-    algorithm.clip_advantages=False \
+    rllm.mask_truncated_samples=False \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='rllm-agent' \
@@ -62,6 +59,6 @@ python3 -m examples.deepscaler.train_deepscaler \
     trainer.save_freq=20 \
     trainer.test_freq=20 \
     trainer.default_hdfs_dir=null \
-    agent.max_steps=1 \
-    agent.use_stepwise_advantage=False \
+    rllm.agent.max_steps=1 \
+    rllm.stepwise_advantage.enable=False \
     trainer.total_epochs=100

examples/deepscaler/train_deepscaler_24k.sh

@@ -30,14 +30,12 @@ python3 -m examples.deepscaler.train_deepscaler \
     actor_rollout_ref.actor.kl_loss_coef=0.001 \
     actor_rollout_ref.actor.kl_loss_type=low_var_kl \
     actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \
-    actor_rollout_ref.actor.grad_norm_threshold=10 \
     actor_rollout_ref.model.enable_gradient_checkpointing=True \
     actor_rollout_ref.actor.fsdp_config.param_offload=True \
     actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \
     actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
     actor_rollout_ref.rollout.name=vllm \
     actor_rollout_ref.rollout.mode="async" \
-    actor_rollout_ref.rollout.chat_scheduler=verl.schedulers.completions_scheduler.CompletionsScheduler \
     actor_rollout_ref.rollout.enforce_eager=False \
     actor_rollout_ref.rollout.temperature=0.6 \
     actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \
@@ -50,8 +48,7 @@ python3 -m examples.deepscaler.train_deepscaler \
     actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
     actor_rollout_ref.actor.entropy_coeff=0 \
     algorithm.kl_ctrl.kl_coef=0.001 \
-    algorithm.mask_truncated_samples=False \
-    algorithm.clip_advantages=False \
+    rllm.mask_truncated_samples=False \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='rllm-agent' \
@@ -62,6 +59,6 @@ python3 -m examples.deepscaler.train_deepscaler \
     trainer.save_freq=20 \
     trainer.test_freq=20 \
     trainer.default_hdfs_dir=null \
-    agent.max_steps=1 \
-    agent.use_stepwise_advantage=False \
+    rllm.agent.max_steps=1 \
+    rllm.stepwise_advantage.enable=False \
     trainer.total_epochs=100

examples/deepscaler/train_deepscaler_8k.sh

@@ -30,14 +30,12 @@ python3 -m examples.deepscaler.train_deepscaler \
     actor_rollout_ref.actor.kl_loss_coef=0.001 \
     actor_rollout_ref.actor.kl_loss_type=low_var_kl \
     actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \
-    actor_rollout_ref.actor.grad_norm_threshold=10 \
     actor_rollout_ref.model.enable_gradient_checkpointing=True \
     actor_rollout_ref.actor.fsdp_config.param_offload=True \
     actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \
     actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
     actor_rollout_ref.rollout.name=vllm \
     actor_rollout_ref.rollout.mode="async" \
-    actor_rollout_ref.rollout.chat_scheduler=verl.schedulers.completions_scheduler.CompletionsScheduler \
     actor_rollout_ref.rollout.enforce_eager=False \
     actor_rollout_ref.rollout.temperature=0.6 \
     actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \
@@ -50,8 +48,7 @@ python3 -m examples.deepscaler.train_deepscaler \
     actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
     actor_rollout_ref.actor.entropy_coeff=0 \
     algorithm.kl_ctrl.kl_coef=0.001 \
-    algorithm.mask_truncated_samples=False \
-    algorithm.clip_advantages=False \
+    rllm.mask_truncated_samples=False \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='rllm-agent' \
@@ -62,6 +59,6 @@ python3 -m examples.deepscaler.train_deepscaler \
     trainer.save_freq=20 \
     trainer.test_freq=20 \
     trainer.default_hdfs_dir=null \
-    agent.max_steps=1 \
-    agent.use_stepwise_advantage=False \
+    rllm.agent.max_steps=1 \
+    rllm.stepwise_advantage.enable=False \
     trainer.total_epochs=100

examples/frozenlake/train_frozenlake_agent.py

@@ -6,7 +6,7 @@
 from rllm.trainer.agent_trainer import AgentTrainer
 
 
-@hydra.main(config_path="pkg://rllm.trainer.config", config_name="ppo_trainer", version_base=None)
+@hydra.main(config_path="pkg://rllm.trainer.config", config_name="agent_ppo_trainer", version_base=None)
 def main(config):
     train_dataset = DatasetRegistry.load_dataset("frozenlake", "train")
     val_dataset = DatasetRegistry.load_dataset("frozenlake", "test")

examples/frozenlake/train_frozenlake_agent.sh

@@ -28,14 +28,12 @@ python3 -m examples.frozenlake.train_frozenlake_agent \
     actor_rollout_ref.actor.kl_loss_coef=0.001 \
     actor_rollout_ref.actor.kl_loss_type=low_var_kl \
     actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \
-    actor_rollout_ref.actor.grad_norm_threshold=10 \
     actor_rollout_ref.model.enable_gradient_checkpointing=True \
     actor_rollout_ref.actor.fsdp_config.param_offload=True \
     actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \
     actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
     actor_rollout_ref.rollout.name=vllm \
     actor_rollout_ref.rollout.mode="async" \
-    actor_rollout_ref.rollout.chat_scheduler=verl.schedulers.completions_scheduler.CompletionsScheduler \
     actor_rollout_ref.rollout.enforce_eager=False \
     actor_rollout_ref.rollout.temperature=0.7 \
     actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \
@@ -49,8 +47,7 @@ python3 -m examples.frozenlake.train_frozenlake_agent \
     actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
     actor_rollout_ref.actor.entropy_coeff=0 \
     algorithm.kl_ctrl.kl_coef=0.001 \
-    algorithm.mask_truncated_samples=False \
-    algorithm.clip_advantages=False \
+    rllm.mask_truncated_samples=False \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='rllm-agent' \
@@ -61,14 +58,13 @@ python3 -m examples.frozenlake.train_frozenlake_agent \
     trainer.save_freq=40 \
     trainer.test_freq=10 \
     trainer.default_hdfs_dir=null \
-    trainer.rejection_sample=True \
-    trainer.rejection_sample_multiplier=2 \
-    +env.env_args.max_steps=8 \
-    +env.env_args.is_slippery=False \
-    agent.max_steps=10 \
-    agent.async_engine=True \
-    agent.use_stepwise_advantage=False \
-    +agent.engine_args.disable_thinking=False \
-    +agent.agent_args.max_steps=10 \
-    +agent.agent_args.use_accumulate_history=True \
+    rllm.rejection_sample.enable=True \
+    rllm.rejection_sample.multiplier=2 \
+    +rllm.env.env_args.max_steps=8 \
+    +rllm.env.env_args.is_slippery=False \
+    rllm.agent.max_steps=10 \
+    rllm.stepwise_advantage.enable=False \
+    rllm.disable_thinking=False \
+    +rllm.agent.agent_args.max_steps=10 \
+    +rllm.agent.agent_args.use_accumulate_history=True \
     trainer.total_epochs=1

examples/math_tool/train_math_with_tool.py

@@ -7,7 +7,7 @@
 from rllm.trainer.agent_trainer import AgentTrainer
 
 
-@hydra.main(config_path="pkg://rllm.trainer.config", config_name="ppo_trainer", version_base=None)
+@hydra.main(config_path="pkg://rllm.trainer.config", config_name="agent_ppo_trainer", version_base=None)
 def main(config):
     train_dataset = DatasetRegistry.load_dataset("deepscaler_math", "train")
     test_dataset = DatasetRegistry.load_dataset("aime2024", "test")