set -x

python3 -m verl.trainer.main_vgpt_ppo \
    actor_rollout_ref.actor.interact=True \
    actor_rollout_ref.rollout.interact=True \
    actor_rollout_ref.rollout.interact_max_tokens=64 \
    processor.tokenizer.path=checkpoints/libero/test/tokenizer  \
    processor.interact=True\
    processor.tokenizer.name=ctx_cnn \
    data.max_prompt_length=1095 \
    data.max_response_length=497 \
    processor.bos_token_id=9006 \
    processor.eos_token_id=9007 \
    processor.pad_token_id=9007 \
    processor.tokens_per_frame=64 \
    processor.processor_type=ctx_msp \
    processor.max_length=1592 \
    processor.action_dim=7 \
    processor.action_ranges_path=vid_wm/ivideogpt/configs/vgpt/libero_action_ranges.pth \
    data.video.segment_length=8 \
    algorithm.adv_estimator=grpo \
    data.train_batch_size=128 \
    data.video.dataset_path=/path/to/your/owndata/data_256 \
    data.video.oxe_data_mixes_type=libero \
    actor_rollout_ref.model.path=checkpoints/libero/backbone/transformed/test \
    actor_rollout_ref.actor.optim.lr=1e-5 \
    actor_rollout_ref.model.use_remove_padding=False \
    actor_rollout_ref.actor.vocab_size=9008 \
    actor_rollout_ref.actor.ppo_mini_batch_size=32 \
    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \
    actor_rollout_ref.actor.use_ce_loss=False \
    actor_rollout_ref.actor.ce_loss_coef=0.001 \
    actor_rollout_ref.actor.use_kl_loss=True \
    actor_rollout_ref.actor.kl_loss_coef=5e-4 \
    actor_rollout_ref.actor.kl_loss_type=low_var_kl \
    actor_rollout_ref.actor.entropy_coeff=0 \
    actor_rollout_ref.model.enable_gradient_checkpointing=False \
    actor_rollout_ref.actor.fsdp_config.param_offload=False \
    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \
    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \
    actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
    actor_rollout_ref.rollout.name=vllm \
    actor_rollout_ref.rollout.top_k=-1 \
    actor_rollout_ref.rollout.temperature=1.5 \
    actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \
    actor_rollout_ref.rollout.n=12  \
    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \
    actor_rollout_ref.ref.fsdp_config.param_offload=True \
    algorithm.use_kl_in_reward=False \
    trainer.reward_fn=mae \
    trainer.critic_warmup=0 \
    trainer.logger=['tensorboard'] \
    trainer.project_name='verl_grpo_example_vgpt' \
    trainer.experiment_name='ctx_vgpt_msp8' \
    trainer.n_gpus_per_node=4 \
    trainer.nnodes=1 \
    trainer.save_freq=200 \
    trainer.val_iters=10 \
    trainer.test_freq=-1 \
    trainer.default_local_dir=checkpoints/libero/rlvr/${date_today}/ckpt_${CKPT_STEPS}_${POST_EXP_NAME} \
    trainer.total_training_steps=400 \
    trainer.msp_reward_aggregate=discount \
    trainer.loss_weight.mse=0 \
    trainer.loss_weight.lpips=1 \
    trainer.loss_weight.mae=1 \

