set -x

# Run with: bash scripts/libero/post_train_rlvr.sh

python3 -m verl.trainer.main_vla_rft_grpo \
    trainer.total_training_steps=400 \
    trainer.save_freq=50 \
    trainer.nnodes=1 \
    trainer.n_gpus_per_node=4 \
    trainer.use_ac_reward=False \
    trainer.ac_reward_type='l1' \
    trainer.reward_fn=mae \
    trainer.logger=['tensorboard'] \
    trainer.project_name='vla_rft' \
    trainer.experiment_name='vla_rft_fm' \
    trainer.save_last_freq=20 \
    trainer.save_last_num=2 \
    trainer.val_iters=10 \
    trainer.test_freq=-1 \
    trainer.default_local_dir=checkpoints/libero/RFT/${LIBERO_TASK_NAME}/${DATE}_${POST_EXP_NAME} \
    trainer.msp_reward_aggregate=mean \
    trainer.msp_reward_discount=0.95 \
    trainer.loss_weight.mse=0 \
    trainer.loss_weight.lpips=1 \
    trainer.loss_weight.mae=1 \
    data.train_batch_size=16 \
    data.video.dataset_path=data/modified_libero_rlds \
    data.video.dataset_name=libero_${LIBERO_TASK_NAME}_no_noops \
    algorithm.adv_estimator=grpo \
    actor_rollout_ref.actor.log_l1_loss=True \
    actor_rollout_ref.actor.optim.lr=1e-6 \
    actor_rollout_ref.actor.optim.sigma_lr=1e-5 \
    actor_rollout_ref.actor.optim.lr_warmup_steps=10 \
    actor_rollout_ref.model.use_remove_padding=False \
    actor_rollout_ref.actor.ppo_mini_batch_size=16 \
    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \
    actor_rollout_ref.actor.use_mse_loss=True \
    actor_rollout_ref.actor.mse_loss_coef=0.01 \
    actor_rollout_ref.actor.mse_kl_low=0 \
    actor_rollout_ref.actor.mse_kl_high=0.2 \
    actor_rollout_ref.actor.entropy_coeff=0.003 \
    actor_rollout_ref.model.enable_gradient_checkpointing=False \
    actor_rollout_ref.actor.fsdp_config.param_offload=False \
    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \
    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \
    actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
    actor_rollout_ref.rollout.name=hf \
    actor_rollout_ref.rollout.n=16 \
    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 \
    actor_rollout_ref.ref.fsdp_config.param_offload=True \
    actor_rollout_ref.model.ckpt_path=checkpoints/libero/Base/${LIBERO_TASK_NAME} \
    actor_rollout_ref.model.cfg_path=checkpoints/libero/Base/${LIBERO_TASK_NAME}/config.json \
    algorithm.use_kl_in_reward=False \
    world_model_rollout.model.path=checkpoints/libero/WorldModel/${LIBERO_TASK_NAME} \
    world_model_rollout.model.use_remove_padding=False \
    world_model_rollout.world_model.vocab_size=9008 \
    world_model_rollout.rollout.tensor_model_parallel_size=1 \
    world_model_rollout.rollout.name=vllm \
    world_model_rollout.rollout.do_sample=True \
    world_model_rollout.rollout.is_validate=True \
    world_model_rollout.rollout.val_kwargs.top_k=-1 \
    world_model_rollout.rollout.val_kwargs.top_p=0.8 \
    world_model_rollout.rollout.val_kwargs.temperature=1.0 \
    world_model_rollout.rollout.gpu_memory_utilization=0.85 \
    world_model_rollout.world_model.interact=True \
    world_model_rollout.rollout.interact=True \
    world_model_rollout.rollout.interact_max_tokens=64 \
    processor.action_dim=7 \
    processor.action_ranges_path=train/verl/ivideogpt/configs/libero_action_ranges.pth \
    processor.tokenizer.path=checkpoints/libero/WorldModel/Tokenizer \
    processor.interact=True\
    processor.tokenizer.name=ctx_cnn \
    data.max_prompt_length=1095 \
    data.max_response_length=568 \
    processor.bos_token_id=9006 \
    processor.eos_token_id=9007 \
    processor.pad_token_id=9007 \
    processor.tokens_per_frame=64 \
    processor.processor_type=ctx_msp \
    processor.max_length=1663 \
    processor.use_img_gt_ac=True \