
#Local attention
USE_LINEAR=1 USE_LOCAL=1 SPLIT=64 CUDA_VISIBLE_DEVICES="0" torchrun --nproc_per_node=1 --master_port=43534 fine-tune.py  --model_name_or_path meta-llama/Llama-2-7b-hf         --bf16 True         --output_dir ./longlora-local-7b      --model_max_length 4096         --use_flash_attn False         --low_rank_training True         --num_train_epochs 1          --per_device_train_batch_size 1             --per_device_eval_batch_size 2             --gradient_accumulation_steps 64             --evaluation_strategy "no"             --save_strategy "steps"             --save_steps 1000             --save_total_limit 2             --learning_rate 2e-5             --weight_decay 0.0             --warmup_steps 20             --lr_scheduler_type "constant_with_warmup"             --logging_steps 1     --deepspeed "ds_configs/stage2.json"  --tf32 True --max_steps 1000 --model_type llama 

#local + alpha * Grouped attention
USE_LINEAR=1 USE_GLOBAL=1 USE_LOCAL=1 GLOBAL_FACTOR=1 CUDA_VISIBLE_DEVICES="0" SPLIT=64 torchrun --nproc_per_node=1 --master_port=35464 fine-tune.py  --model_name_or_path meta-llama/Llama-2-7b-hf         --bf16 True         --output_dir ./longlora-global-7b         --model_max_length 4096         --use_flash_attn False         --low_rank_training True         --num_train_epochs 1          --per_device_train_batch_size 8             --per_device_eval_batch_size 2             --gradient_accumulation_steps 8             --evaluation_strategy "no"             --save_strategy "steps"             --save_steps 1000             --save_total_limit 2             --learning_rate 2e-5             --weight_decay 0.0             --warmup_steps 20             --lr_scheduler_type "constant_with_warmup"             --logging_steps 1     --deepspeed "ds_configs/stage2.json"  --tf32 True --max_steps 1000 --model_type llama 


#Local + Conv
USE_LINEAR=1 USE_LOCAL=1 ZERO_CONV=1 ADD_CONV=1 CUDA_VISIBLE_DEVICES="0" SPLIT=64 torchrun --nproc_per_node=1 --master_port=43534 fine-tune.py  --model_name_or_path meta-llama/Llama-2-7b-hf         --bf16 True         --output_dir ./local-addconv-7b                --model_max_length 4096         --use_flash_attn False         --low_rank_training True         --num_train_epochs 1          --per_device_train_batch_size 1             --per_device_eval_batch_size 2             --gradient_accumulation_steps 64             --evaluation_strategy "no"             --save_strategy "steps"             --save_steps 1000             --save_total_limit 2             --learning_rate 2e-5             --weight_decay 0.0             --warmup_steps 20             --lr_scheduler_type "constant_with_warmup"             --logging_steps 1     --deepspeed "ds_configs/stage2.json"  --tf32 True --max_steps 1000 --model_type llama


#Local + alpha * global + Conv (ours)
USE_LINEAR=1 USE_GLOBAL=1 USE_LOCAL=1 GLOBAL_FACTOR=1 ZERO_CONV=1 ADD_CONV=1 CUDA_VISIBLE_DEVICES="0" SPLIT=64 torchrun --nproc_per_node=1 --master_port=34242 fine-tune.py  --model_name_or_path meta-llama/Llama-2-7b-hf          --bf16 True         --output_dir ./both-addconv-7b          --model_max_length 4096         --use_flash_attn False         --low_rank_training True         --num_train_epochs 1          --per_device_train_batch_size 1             --per_device_eval_batch_size 2             --gradient_accumulation_steps 64             --evaluation_strategy "no"             --save_strategy "steps"             --save_steps 1000             --save_total_limit 2             --learning_rate 2e-5             --weight_decay 0.0             --warmup_steps 20             --lr_scheduler_type "constant_with_warmup"             --logging_steps 1     --deepspeed "ds_configs/stage2.json"  --tf32 True --max_steps 1000 --model_type llama 
