deepspeed --master_port=10000 --include localhost:0,1,2,3,4,5,6,7 geollama/training/train_mem.py \
--deepspeed scripts/zero2.json \
--model_id meta-llama/Llama-3.2-11B-Vision-Instruct \
--checkpoint_path path_to_pre_trained_checkppoint \
--data_path path_to_instruction_finetune_json \
--image_folder path_to_chatearthnet_folder \
--disable_flash_attn2 True \
--lora_enable True \
--tune_img_projector True \
--tune_ms_adapter True \
--freeze_vision_tower True \
--vision_lora False \
--freeze_llm True \
--bf16 True \
--output_dir output/pre-train \
--num_train_epochs 5  \
--per_device_train_batch_size 16 \
--gradient_accumulation_steps 1 \
--learning_rate 2e-5 \
--projector_lr 2e-5 \
--vision_lr 2e-5 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--adam_beta2 0.95 \
--lr_scheduler_type "cosine" \
--logging_steps 20 \
--tf32 True \
--gradient_checkpointing True \
--report_to wandb \
--lazy_preprocess True \
--save_strategy "steps" \
--resume_training "" \
--save_steps 200 \
--save_total_limit 5 \
--dataloader_num_workers 8
