# export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:32
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
export OMP_NUM_THREADS=8
# export NCCL_IB_DISABLE=0
#####################
# set infiniband device
# export NCCL_IB_HCA=mlx5_1
# export NCCL_IB_GID_INDEX=7
# export NCCL_SOCKET_IFNAME=eth1
# export NCCL_NET_GDR_LEVEL=2
#####################
export NCCL_DEBUG=INFO


VISION_MODEL_VERSION="openai/clip-vit-large-patch14-336"
VISION_MODEL_VERSION_CLEAN="${VISION_MODEL_VERSION//\//_}"
PROMPT_VERSION=qwen_1_5

###########
# export NNODES=2
export NNODES=1
export NUM_GPUS=8
export MASTER_ADDR="127.0.0.1"
export MASTER_PORT=29599
export WORLD_SIZE=$((NNODES * NUM_GPUS))
export RANK=0
###########
NUM_TRAIN_EPOCHS=1
MAX_TOKENS=24
FIX="max${MAX_TOKENS}_"
RUN_NAME="LongVA-continue-sft-3_${FIX}576grid_e${NUM_TRAIN_EPOCHS}"
JSON_PATH="/fs-computility/ai4sData/earth-shared/cms/xlrs_train/ft3.json"
IMAGE_FOLDER="/fs-computility/ai4sData/earth-shared/cms/xlrs_train/jpg_images"

CKPT_PATH=lmms-lab/LongVA-7B

ACCELERATE_CPU_AFFINITY=1 torchrun --nproc_per_node="${NUM_GPUS}" --nnodes="${NNODES}" --node_rank="${RANK}" --master_addr="${MASTER_ADDR}" --master_port="${MASTER_PORT}" \
    longva/train/train_mem.py \
    --deepspeed scripts/zero2_tp2.json \
    --model_name_or_path $CKPT_PATH \
    --version ${PROMPT_VERSION} \
    --data_path=${JSON_PATH} \
    --image_folder ${IMAGE_FOLDER} \
    --mm_tunable_parts="mm_vision_tower,mm_mlp_adapter,mm_language_model" \
    --mm_vision_tower_lr=1e-6 \
    --vision_tower ${VISION_MODEL_VERSION} \
    --mm_projector_type mlp2x_gelu \
    --mm_vision_select_layer -2 \
    --mm_use_im_start_end False \
    --mm_use_im_patch_token False \
    --group_by_modality_length True \
    --image_aspect_ratio anyres \
    --image_grid_pinpoints "[(336, 672), (336, 1008), (336, 1344), (336, 1680), (336, 2016), (336, 2352), (336, 2688), (336, 3024), (336, 3360), (336, 3696), (336, 4032), (336, 4368), (336, 4704), (336, 5040), (336, 5376), (336, 5712), (336, 6048), (336, 6384), (336, 6720), (336, 7056), (336, 7392), (336, 7728), (336, 8064), (336, 8400), (336, 8736), (336, 9072), (336, 9408), (336, 9744), (336, 10080), (336, 10416), (336, 10752), (336, 11088), (336, 11424), (336, 11760), (336, 12096), (336, 12432), (336, 12768), (336, 13104), (336, 13440), (336, 13776), (336, 14112), (336, 14448), (336, 14784), (336, 15120), (336, 15456), (336, 15792), (336, 16128), (336, 16464), (672, 336), (672, 672), (672, 1008), (672, 1344), (672, 1680), (672, 2016), (672, 2352), (672, 2688), (672, 3024), (672, 3360), (672, 3696), (672, 4032), (672, 4368), (672, 4704), (672, 5040), (672, 5376), (672, 5712), (672, 6048), (672, 6384), (672, 6720), (672, 7056), (672, 7392), (672, 7728), (672, 8064), (672, 8400), (672, 8736), (672, 9072), (672, 9408), (672, 9744), (672, 10080), (672, 10416), (672, 10752), (672, 11088), (672, 11424), (672, 11760), (672, 12096), (672, 12432), (672, 12768), (672, 13104), (672, 13440), (672, 13776), (672, 14112), (672, 14448), (672, 14784), (672, 15120), (672, 15456), (672, 15792), (672, 16128), (672, 16464), (672, 16800), (672, 17136), (672, 17472), (672, 17808), (672, 18144), (672, 18480), (672, 18816), (672, 19152), (672, 19488), (672, 19824), (672, 20160), (672, 20496), (672, 20832), (672, 21168), (672, 21504), (672, 21840), (672, 22176), (672, 22512), (672, 22848), (672, 23184), (672, 23520), (672, 23856), (672, 24192), (672, 24528), (672, 24864), (672, 25200), (672, 25536), (672, 25872), (672, 26208), (672, 26544), (672, 26880), (672, 27216), (672, 27552), (672, 27888), (672, 28224), (672, 28560), (672, 28896), (672, 29232), (672, 29568), (672, 29904), (672, 30240), (672, 30576), (672, 30912), (672, 31248), (672, 31584), (672, 31920), (672, 32256), (672, 32592), (672, 32928), (1008, 336), (1008, 672), (1008, 1008), (1008, 1344), (1008, 1680), (1008, 2016), (1008, 2352), (1008, 2688), (1008, 3024), (1008, 3360), (1008, 3696), (1008, 4032), (1008, 4368), (1008, 4704), (1008, 5040), (1008, 5376), (1008, 5712), (1008, 6048), (1008, 6384), (1008, 6720), (1008, 7056), (1008, 7392), (1008, 7728), (1008, 8064), (1008, 8400), (1008, 8736), (1008, 9072), (1008, 9408), (1008, 9744), (1008, 10080), (1008, 10416), (1008, 10752), (1008, 11088), (1008, 11424), (1008, 11760), (1008, 12096), (1008, 12432), (1008, 12768), (1008, 13104), (1008, 13440), (1008, 13776), (1008, 14112), (1008, 14448), (1008, 14784), (1008, 15120), (1008, 15456), (1008, 15792), (1008, 16128), (1008, 16464), (1008, 16800), (1008, 17136), (1008, 17472), (1008, 17808), (1008, 18144), (1008, 18480), (1008, 18816), (1008, 19152), (1008, 19488), (1008, 19824), (1008, 20160), (1008, 20496), (1008, 20832), (1008, 21168), (1008, 21504), (1008, 21840), (1008, 22176), (1008, 22512), (1008, 22848), (1008, 23184), (1008, 23520), (1008, 23856), (1008, 24192), (1008, 24528), (1008, 24864), (1008, 25200), (1008, 25536), (1008, 25872), (1008, 26208), (1008, 26544), (1008, 26880), (1008, 27216), (1008, 27552), (1008, 27888), (1008, 28224), (1008, 28560), (1008, 28896), (1008, 29232), (1008, 29568), (1008, 29904), (1008, 30240), (1008, 30576), (1008, 30912), (1008, 31248), (1008, 31584), (1008, 31920), (1008, 32256), (1008, 32592), (1008, 32928), (1008, 33264), (1008, 33600), (1008, 33936), (1008, 34272), (1008, 34608), (1008, 34944), (1008, 35280), (1008, 35616), (1008, 35952), (1008, 36288), (1008, 36624), (1008, 36960), (1008, 37296), (1008, 37632), (1008, 37968), (1008, 38304), (1008, 38640), (1008, 38976), (1008, 39312), (1008, 39648), (1008, 39984), (1008, 40320), (1008, 40656), (1008, 40992), (1008, 41328), (1008, 41664), (1008, 42000), (1008, 42336), (1008, 42672), (1008, 43008), (1008, 43344), (1008, 43680), (1008, 44016), (1008, 44352), (1008, 44688), (1008, 45024), (1008, 45360), (1008, 45696), (1008, 46032), (1008, 46368), (1008, 46704), (1008, 47040), (1008, 47376), (1008, 47712), (1008, 48048), (1008, 48384), (1008, 48720), (1008, 49056), (1008, 49392), (1344, 336), (1344, 672), (1344, 1008), (1344, 1344), (1344, 1680), (1344, 2016), (1344, 2352), (1344, 2688), (1344, 3024), (1344, 3360), (1344, 3696), (1344, 4032), (1344, 4368), (1344, 4704), (1344, 5040), (1344, 5376), (1344, 5712), (1344, 6048), (1344, 6384), (1344, 6720), (1344, 7056), (1344, 7392), (1344, 7728), (1344, 8064), (1344, 8400), (1344, 8736), (1344, 9072), (1344, 9408), (1344, 9744), (1344, 10080), (1344, 10416), (1344, 10752), (1344, 11088), (1344, 11424), (1344, 11760), (1344, 12096), (1344, 12432), (1344, 12768), (1344, 13104), (1344, 13440), (1344, 13776), (1344, 14112), (1344, 14448), (1344, 14784), (1344, 15120), (1344, 15456), (1344, 15792), (1344, 16128), (1344, 16464), (1344, 16800), (1344, 17136), (1344, 17472), (1344, 17808), (1344, 18144), (1344, 18480), (1344, 18816), (1344, 19152), (1344, 19488), (1344, 19824), (1344, 20160), (1344, 20496), (1344, 20832), (1344, 21168), (1344, 21504), (1344, 21840), (1344, 22176), (1344, 22512), (1344, 22848), (1344, 23184), (1344, 23520), (1344, 23856), (1344, 24192), (1344, 24528), (1344, 24864), (1344, 25200), (1344, 25536), (1344, 25872), (1344, 26208), (1344, 26544), (1344, 26880), (1344, 27216), (1344, 27552), (1344, 27888), (1344, 28224), (1344, 28560), (1344, 28896), (1344, 29232), (1344, 29568), (1344, 29904), (1344, 30240), (1344, 30576), (1344, 30912), (1344, 31248), (1344, 31584), (1344, 31920), (1344, 32256), (1344, 32592), (1344, 32928), (1344, 33264), (1344, 33600), (1344, 33936), (1344, 34272), (1344, 34608), (1344, 34944), (1344, 35280), (1344, 35616), (1344, 35952), (1344, 36288), (1344, 36624), (1344, 36960), (1344, 37296), (1344, 37632), (1344, 37968), (1344, 38304), (1344, 38640), (1344, 38976), (1344, 39312), (1344, 39648), (1344, 39984), (1344, 40320), (1344, 40656), (1344, 40992), (1344, 41328), (1344, 41664), (1344, 42000), (1344, 42336), (1344, 42672), (1344, 43008), (1344, 43344), (1344, 43680), (1344, 44016), (1344, 44352), (1344, 44688), (1344, 45024), (1344, 45360), (1344, 45696), (1344, 46032), (1344, 46368), (1344, 46704), (1344, 47040), (1344, 47376), (1344, 47712), (1344, 48048), (1344, 48384), (1680, 336), (1680, 672), (1680, 1008), (1680, 1344), (1680, 1680), (1680, 2016), (1680, 2352), (1680, 2688), (1680, 3024), (1680, 3360), (1680, 3696), (1680, 4032), (1680, 4368), (1680, 4704), (1680, 5040), (1680, 5376), (1680, 5712), (1680, 6048), (1680, 6384), (1680, 6720), (1680, 7056), (1680, 7392), (1680, 7728), (1680, 8064), (1680, 8400), (1680, 8736), (1680, 9072), (1680, 9408), (1680, 9744), (1680, 10080), (1680, 10416), (1680, 10752), (1680, 11088), (1680, 11424), (1680, 11760), (1680, 12096), (1680, 12432), (1680, 12768), (1680, 13104), (1680, 13440), (1680, 13776), (1680, 14112), (1680, 14448), (1680, 14784), (1680, 15120), (1680, 15456), (1680, 15792), (1680, 16128), (1680, 16464), (1680, 16800), (1680, 17136), (1680, 17472), (1680, 17808), (1680, 18144), (1680, 18480), (1680, 18816), (1680, 19152), (1680, 19488), (1680, 19824), (1680, 20160), (1680, 20496), (1680, 20832), (1680, 21168), (1680, 21504), (1680, 21840), (1680, 22176), (1680, 22512), (1680, 22848), (1680, 23184), (1680, 23520), (1680, 23856), (1680, 24192), (1680, 24528), (1680, 24864), (1680, 25200), (1680, 25536), (1680, 25872), (1680, 26208), (1680, 26544), (1680, 26880), (1680, 27216), (1680, 27552), (1680, 27888), (1680, 28224), (1680, 28560), (1680, 28896), (1680, 29232), (1680, 29568), (1680, 29904), (1680, 30240), (1680, 30576), (1680, 30912), (1680, 31248), (1680, 31584), (1680, 31920), (1680, 32256), (1680, 32592), (1680, 32928), (1680, 33264), (1680, 33600), (1680, 33936), (1680, 34272), (1680, 34608), (1680, 34944), (1680, 35280), (1680, 35616), (1680, 35952), (1680, 36288), (1680, 36624), (1680, 36960), (1680, 37296), (1680, 37632), (1680, 37968), (1680, 38304), (1680, 38640), (2016, 336), (2016, 672), (2016, 1008), (2016, 1344), (2016, 1680), (2016, 2016), (2016, 2352), (2016, 2688), (2016, 3024), (2016, 3360), (2016, 3696), (2016, 4032), (2016, 4368), (2016, 4704), (2016, 5040), (2016, 5376), (2016, 5712), (2016, 6048), (2016, 6384), (2016, 6720), (2016, 7056), (2016, 7392), (2016, 7728), (2016, 8064), (2016, 8400), (2016, 8736), (2016, 9072), (2016, 9408), (2016, 9744), (2016, 10080), (2016, 10416), (2016, 10752), (2016, 11088), (2016, 11424), (2016, 11760), (2016, 12096), (2016, 12432), (2016, 12768), (2016, 13104), (2016, 13440), (2016, 13776), (2016, 14112), (2016, 14448), (2016, 14784), (2016, 15120), (2016, 15456), (2016, 15792), (2016, 16128), (2016, 16464), (2016, 16800), (2016, 17136), (2016, 17472), (2016, 17808), (2016, 18144), (2016, 18480), (2016, 18816), (2016, 19152), (2016, 19488), (2016, 19824), (2016, 20160), (2016, 20496), (2016, 20832), (2016, 21168), (2016, 21504), (2016, 21840), (2016, 22176), (2016, 22512), (2016, 22848), (2016, 23184), (2016, 23520), (2016, 23856), (2016, 24192), (2016, 24528), (2016, 24864), (2016, 25200), (2016, 25536), (2016, 25872), (2016, 26208), (2016, 26544), (2016, 26880), (2016, 27216), (2016, 27552), (2016, 27888), (2016, 28224), (2016, 28560), (2016, 28896), (2016, 29232), (2016, 29568), (2016, 29904), (2016, 30240), (2016, 30576), (2016, 30912), (2016, 31248), (2016, 31584), (2016, 31920), (2016, 32256), (2352, 336), (2352, 672), (2352, 1008), (2352, 1344), (2352, 1680), (2352, 2016), (2352, 2352), (2352, 2688), (2352, 3024), (2352, 3360), (2352, 3696), (2352, 4032), (2352, 4368), (2352, 4704), (2352, 5040), (2352, 5376), (2352, 5712), (2352, 6048), (2352, 6384), (2352, 6720), (2352, 7056), (2352, 7392), (2352, 7728), (2352, 8064), (2352, 8400), (2352, 8736), (2352, 9072), (2352, 9408), (2352, 9744), (2352, 10080), (2352, 10416), (2352, 10752), (2352, 11088), (2352, 11424), (2352, 11760), (2352, 12096), (2352, 12432), (2352, 12768), (2352, 13104), (2352, 13440), (2352, 13776), (2352, 14112), (2352, 14448), (2352, 14784), (2352, 15120), (2352, 15456), (2352, 15792), (2352, 16128), (2352, 16464), (2352, 16800), (2352, 17136), (2352, 17472), (2352, 17808), (2352, 18144), (2352, 18480), (2352, 18816), (2352, 19152), (2352, 19488), (2352, 19824), (2352, 20160), (2352, 20496), (2352, 20832), (2352, 21168), (2352, 21504), (2352, 21840), (2352, 22176), (2352, 22512), (2352, 22848), (2352, 23184), (2352, 23520), (2352, 23856), (2352, 24192), (2352, 24528), (2352, 24864), (2352, 25200), (2352, 25536), (2352, 25872), (2352, 26208), (2352, 26544), (2352, 26880), (2352, 27216), (2352, 27552), (2688, 336), (2688, 672), (2688, 1008), (2688, 1344), (2688, 1680), (2688, 2016), (2688, 2352), (2688, 2688), (2688, 3024), (2688, 3360), (2688, 3696), (2688, 4032), (2688, 4368), (2688, 4704), (2688, 5040), (2688, 5376), (2688, 5712), (2688, 6048), (2688, 6384), (2688, 6720), (2688, 7056), (2688, 7392), (2688, 7728), (2688, 8064), (2688, 8400), (2688, 8736), (2688, 9072), (2688, 9408), (2688, 9744), (2688, 10080), (2688, 10416), (2688, 10752), (2688, 11088), (2688, 11424), (2688, 11760), (2688, 12096), (2688, 12432), (2688, 12768), (2688, 13104), (2688, 13440), (2688, 13776), (2688, 14112), (2688, 14448), (2688, 14784), (2688, 15120), (2688, 15456), (2688, 15792), (2688, 16128), (2688, 16464), (2688, 16800), (2688, 17136), (2688, 17472), (2688, 17808), (2688, 18144), (2688, 18480), (2688, 18816), (2688, 19152), (2688, 19488), (2688, 19824), (2688, 20160), (2688, 20496), (2688, 20832), (2688, 21168), (2688, 21504), (2688, 21840), (2688, 22176), (2688, 22512), (2688, 22848), (2688, 23184), (2688, 23520), (2688, 23856), (2688, 24192), (3024, 336), (3024, 672), (3024, 1008), (3024, 1344), (3024, 1680), (3024, 2016), (3024, 2352), (3024, 2688), (3024, 3024), (3024, 3360), (3024, 3696), (3024, 4032), (3024, 4368), (3024, 4704), (3024, 5040), (3024, 5376), (3024, 5712), (3024, 6048), (3024, 6384), (3024, 6720), (3024, 7056), (3024, 7392), (3024, 7728), (3024, 8064), (3024, 8400), (3024, 8736), (3024, 9072), (3024, 9408), (3024, 9744), (3024, 10080), (3024, 10416), (3024, 10752), (3024, 11088), (3024, 11424), (3024, 11760), (3024, 12096), (3024, 12432), (3024, 12768), (3024, 13104), (3024, 13440), (3024, 13776), (3024, 14112), (3024, 14448), (3024, 14784), (3024, 15120), (3024, 15456), (3024, 15792), (3024, 16128), (3024, 16464), (3024, 16800), (3024, 17136), (3024, 17472), (3024, 17808), (3024, 18144), (3024, 18480), (3024, 18816), (3024, 19152), (3024, 19488), (3024, 19824), (3024, 20160), (3024, 20496), (3024, 20832), (3024, 21168), (3024, 21504), (3360, 336), (3360, 672), (3360, 1008), (3360, 1344), (3360, 1680), (3360, 2016), (3360, 2352), (3360, 2688), (3360, 3024), (3360, 3360), (3360, 3696), (3360, 4032), (3360, 4368), (3360, 4704), (3360, 5040), (3360, 5376), (3360, 5712), (3360, 6048), (3360, 6384), (3360, 6720), (3360, 7056), (3360, 7392), (3360, 7728), (3360, 8064), (3360, 8400), (3360, 8736), (3360, 9072), (3360, 9408), (3360, 9744), (3360, 10080), (3360, 10416), (3360, 10752), (3360, 11088), (3360, 11424), (3360, 11760), (3360, 12096), (3360, 12432), (3360, 12768), (3360, 13104), (3360, 13440), (3360, 13776), (3360, 14112), (3360, 14448), (3360, 14784), (3360, 15120), (3360, 15456), (3360, 15792), (3360, 16128), (3360, 16464), (3360, 16800), (3360, 17136), (3360, 17472), (3360, 17808), (3360, 18144), (3360, 18480), (3360, 18816), (3360, 19152), (3696, 336), (3696, 672), (3696, 1008), (3696, 1344), (3696, 1680), (3696, 2016), (3696, 2352), (3696, 2688), (3696, 3024), (3696, 3360), (3696, 3696), (3696, 4032), (3696, 4368), (3696, 4704), (3696, 5040), (3696, 5376), (3696, 5712), (3696, 6048), (3696, 6384), (3696, 6720), (3696, 7056), (3696, 7392), (3696, 7728), (3696, 8064), (3696, 8400), (3696, 8736), (3696, 9072), (3696, 9408), (3696, 9744), (3696, 10080), (3696, 10416), (3696, 10752), (3696, 11088), (3696, 11424), (3696, 11760), (3696, 12096), (3696, 12432), (3696, 12768), (3696, 13104), (3696, 13440), (3696, 13776), (3696, 14112), (3696, 14448), (3696, 14784), (3696, 15120), (3696, 15456), (3696, 15792), (3696, 16128), (3696, 16464), (3696, 16800), (3696, 17136), (3696, 17472), (4032, 336), (4032, 672), (4032, 1008), (4032, 1344), (4032, 1680), (4032, 2016), (4032, 2352), (4032, 2688), (4032, 3024), (4032, 3360), (4032, 3696), (4032, 4032), (4032, 4368), (4032, 4704), (4032, 5040), (4032, 5376), (4032, 5712), (4032, 6048), (4032, 6384), (4032, 6720), (4032, 7056), (4032, 7392), (4032, 7728), (4032, 8064), (4032, 8400), (4032, 8736), (4032, 9072), (4032, 9408), (4032, 9744), (4032, 10080), (4032, 10416), (4032, 10752), (4032, 11088), (4032, 11424), (4032, 11760), (4032, 12096), (4032, 12432), (4032, 12768), (4032, 13104), (4032, 13440), (4032, 13776), (4032, 14112), (4032, 14448), (4032, 14784), (4032, 15120), (4032, 15456), (4032, 15792), (4032, 16128), (4368, 336), (4368, 672), (4368, 1008), (4368, 1344), (4368, 1680), (4368, 2016), (4368, 2352), (4368, 2688), (4368, 3024), (4368, 3360), (4368, 3696), (4368, 4032), (4368, 4368), (4368, 4704), (4368, 5040), (4368, 5376), (4368, 5712), (4368, 6048), (4368, 6384), (4368, 6720), (4368, 7056), (4368, 7392), (4368, 7728), (4368, 8064), (4368, 8400), (4368, 8736), (4368, 9072), (4368, 9408), (4368, 9744), (4368, 10080), (4368, 10416), (4368, 10752), (4368, 11088), (4368, 11424), (4368, 11760), (4368, 12096), (4368, 12432), (4368, 12768), (4368, 13104), (4368, 13440), (4368, 13776), (4368, 14112), (4368, 14448), (4368, 14784), (4704, 336), (4704, 672), (4704, 1008), (4704, 1344), (4704, 1680), (4704, 2016), (4704, 2352), (4704, 2688), (4704, 3024), (4704, 3360), (4704, 3696), (4704, 4032), (4704, 4368), (4704, 4704), (4704, 5040), (4704, 5376), (4704, 5712), (4704, 6048), (4704, 6384), (4704, 6720), (4704, 7056), (4704, 7392), (4704, 7728), (4704, 8064), (4704, 8400), (4704, 8736), (4704, 9072), (4704, 9408), (4704, 9744), (4704, 10080), (4704, 10416), (4704, 10752), (4704, 11088), (4704, 11424), (4704, 11760), (4704, 12096), (4704, 12432), (4704, 12768), (4704, 13104), (4704, 13440), (4704, 13776), (5040, 336), (5040, 672), (5040, 1008), (5040, 1344), (5040, 1680), (5040, 2016), (5040, 2352), (5040, 2688), (5040, 3024), (5040, 3360), (5040, 3696), (5040, 4032), (5040, 4368), (5040, 4704), (5040, 5040), (5040, 5376), (5040, 5712), (5040, 6048), (5040, 6384), (5040, 6720), (5040, 7056), (5040, 7392), (5040, 7728), (5040, 8064), (5040, 8400), (5040, 8736), (5040, 9072), (5040, 9408), (5040, 9744), (5040, 10080), (5040, 10416), (5040, 10752), (5040, 11088), (5040, 11424), (5040, 11760), (5040, 12096), (5040, 12432), (5040, 12768), (5376, 336), (5376, 672), (5376, 1008), (5376, 1344), (5376, 1680), (5376, 2016), (5376, 2352), (5376, 2688), (5376, 3024), (5376, 3360), (5376, 3696), (5376, 4032), (5376, 4368), (5376, 4704), (5376, 5040), (5376, 5376), (5376, 5712), (5376, 6048), (5376, 6384), (5376, 6720), (5376, 7056), (5376, 7392), (5376, 7728), (5376, 8064), (5376, 8400), (5376, 8736), (5376, 9072), (5376, 9408), (5376, 9744), (5376, 10080), (5376, 10416), (5376, 10752), (5376, 11088), (5376, 11424), (5376, 11760), (5376, 12096), (5712, 336), (5712, 672), (5712, 1008), (5712, 1344), (5712, 1680), (5712, 2016), (5712, 2352), (5712, 2688), (5712, 3024), (5712, 3360), (5712, 3696), (5712, 4032), (5712, 4368), (5712, 4704), (5712, 5040), (5712, 5376), (5712, 5712), (5712, 6048), (5712, 6384), (5712, 6720), (5712, 7056), (5712, 7392), (5712, 7728), (5712, 8064), (5712, 8400), (5712, 8736), (5712, 9072), (5712, 9408), (5712, 9744), (5712, 10080), (5712, 10416), (5712, 10752), (5712, 11088), (6048, 336), (6048, 672), (6048, 1008), (6048, 1344), (6048, 1680), (6048, 2016), (6048, 2352), (6048, 2688), (6048, 3024), (6048, 3360), (6048, 3696), (6048, 4032), (6048, 4368), (6048, 4704), (6048, 5040), (6048, 5376), (6048, 5712), (6048, 6048), (6048, 6384), (6048, 6720), (6048, 7056), (6048, 7392), (6048, 7728), (6048, 8064), (6048, 8400), (6048, 8736), (6048, 9072), (6048, 9408), (6048, 9744), (6048, 10080), (6048, 10416), (6048, 10752), (6384, 336), (6384, 672), (6384, 1008), (6384, 1344), (6384, 1680), (6384, 2016), (6384, 2352), (6384, 2688), (6384, 3024), (6384, 3360), (6384, 3696), (6384, 4032), (6384, 4368), (6384, 4704), (6384, 5040), (6384, 5376), (6384, 5712), (6384, 6048), (6384, 6384), (6384, 6720), (6384, 7056), (6384, 7392), (6384, 7728), (6384, 8064), (6384, 8400), (6384, 8736), (6384, 9072), (6384, 9408), (6384, 9744), (6384, 10080), (6720, 336), (6720, 672), (6720, 1008), (6720, 1344), (6720, 1680), (6720, 2016), (6720, 2352), (6720, 2688), (6720, 3024), (6720, 3360), (6720, 3696), (6720, 4032), (6720, 4368), (6720, 4704), (6720, 5040), (6720, 5376), (6720, 5712), (6720, 6048), (6720, 6384), (6720, 6720), (6720, 7056), (6720, 7392), (6720, 7728), (6720, 8064), (6720, 8400), (6720, 8736), (6720, 9072), (6720, 9408), (7056, 336), (7056, 672), (7056, 1008), (7056, 1344), (7056, 1680), (7056, 2016), (7056, 2352), (7056, 2688), (7056, 3024), (7056, 3360), (7056, 3696), (7056, 4032), (7056, 4368), (7056, 4704), (7056, 5040), (7056, 5376), (7056, 5712), (7056, 6048), (7056, 6384), (7056, 6720), (7056, 7056), (7056, 7392), (7056, 7728), (7056, 8064), (7056, 8400), (7056, 8736), (7056, 9072), (7392, 336), (7392, 672), (7392, 1008), (7392, 1344), (7392, 1680), (7392, 2016), (7392, 2352), (7392, 2688), (7392, 3024), (7392, 3360), (7392, 3696), (7392, 4032), (7392, 4368), (7392, 4704), (7392, 5040), (7392, 5376), (7392, 5712), (7392, 6048), (7392, 6384), (7392, 6720), (7392, 7056), (7392, 7392), (7392, 7728), (7392, 8064), (7392, 8400), (7392, 8736), (7728, 336), (7728, 672), (7728, 1008), (7728, 1344), (7728, 1680), (7728, 2016), (7728, 2352), (7728, 2688), (7728, 3024), (7728, 3360), (7728, 3696), (7728, 4032), (7728, 4368), (7728, 4704), (7728, 5040), (7728, 5376), (7728, 5712), (7728, 6048), (7728, 6384), (7728, 6720), (7728, 7056), (7728, 7392), (7728, 7728), (7728, 8064), (7728, 8400), (8064, 336), (8064, 672), (8064, 1008), (8064, 1344), (8064, 1680), (8064, 2016), (8064, 2352), (8064, 2688), (8064, 3024), (8064, 3360), (8064, 3696), (8064, 4032), (8064, 4368), (8064, 4704), (8064, 5040), (8064, 5376), (8064, 5712), (8064, 6048), (8064, 6384), (8064, 6720), (8064, 7056), (8064, 7392), (8064, 7728), (8064, 8064), (8400, 336), (8400, 672), (8400, 1008), (8400, 1344), (8400, 1680), (8400, 2016), (8400, 2352), (8400, 2688), (8400, 3024), (8400, 3360), (8400, 3696), (8400, 4032), (8400, 4368), (8400, 4704), (8400, 5040), (8400, 5376), (8400, 5712), (8400, 6048), (8400, 6384), (8400, 6720), (8400, 7056), (8400, 7392), (8400, 7728), (8736, 336), (8736, 672), (8736, 1008), (8736, 1344), (8736, 1680), (8736, 2016), (8736, 2352), (8736, 2688), (8736, 3024), (8736, 3360), (8736, 3696), (8736, 4032), (8736, 4368), (8736, 4704), (8736, 5040), (8736, 5376), (8736, 5712), (8736, 6048), (8736, 6384), (8736, 6720), (8736, 7056), (8736, 7392), (9072, 336), (9072, 672), (9072, 1008), (9072, 1344), (9072, 1680), (9072, 2016), (9072, 2352), (9072, 2688), (9072, 3024), (9072, 3360), (9072, 3696), (9072, 4032), (9072, 4368), (9072, 4704), (9072, 5040), (9072, 5376), (9072, 5712), (9072, 6048), (9072, 6384), (9072, 6720), (9072, 7056), (9408, 336), (9408, 672), (9408, 1008), (9408, 1344), (9408, 1680), (9408, 2016), (9408, 2352), (9408, 2688), (9408, 3024), (9408, 3360), (9408, 3696), (9408, 4032), (9408, 4368), (9408, 4704), (9408, 5040), (9408, 5376), (9408, 5712), (9408, 6048), (9408, 6384), (9408, 6720), (9744, 336), (9744, 672), (9744, 1008), (9744, 1344), (9744, 1680), (9744, 2016), (9744, 2352), (9744, 2688), (9744, 3024), (9744, 3360), (9744, 3696), (9744, 4032), (9744, 4368), (9744, 4704), (9744, 5040), (9744, 5376), (9744, 5712), (9744, 6048), (9744, 6384), (10080, 336), (10080, 672), (10080, 1008), (10080, 1344), (10080, 1680), (10080, 2016), (10080, 2352), (10080, 2688), (10080, 3024), (10080, 3360), (10080, 3696), (10080, 4032), (10080, 4368), (10080, 4704), (10080, 5040), (10080, 5376), (10080, 5712), (10080, 6048), (10080, 6384), (10416, 336), (10416, 672), (10416, 1008), (10416, 1344), (10416, 1680), (10416, 2016), (10416, 2352), (10416, 2688), (10416, 3024), (10416, 3360), (10416, 3696), (10416, 4032), (10416, 4368), (10416, 4704), (10416, 5040), (10416, 5376), (10416, 5712), (10416, 6048), (10752, 336), (10752, 672), (10752, 1008), (10752, 1344), (10752, 1680), (10752, 2016), (10752, 2352), (10752, 2688), (10752, 3024), (10752, 3360), (10752, 3696), (10752, 4032), (10752, 4368), (10752, 4704), (10752, 5040), (10752, 5376), (10752, 5712), (10752, 6048), (11088, 336), (11088, 672), (11088, 1008), (11088, 1344), (11088, 1680), (11088, 2016), (11088, 2352), (11088, 2688), (11088, 3024), (11088, 3360), (11088, 3696), (11088, 4032), (11088, 4368), (11088, 4704), (11088, 5040), (11088, 5376), (11088, 5712), (11424, 336), (11424, 672), (11424, 1008), (11424, 1344), (11424, 1680), (11424, 2016), (11424, 2352), (11424, 2688), (11424, 3024), (11424, 3360), (11424, 3696), (11424, 4032), (11424, 4368), (11424, 4704), (11424, 5040), (11424, 5376), (11760, 336), (11760, 672), (11760, 1008), (11760, 1344), (11760, 1680), (11760, 2016), (11760, 2352), (11760, 2688), (11760, 3024), (11760, 3360), (11760, 3696), (11760, 4032), (11760, 4368), (11760, 4704), (11760, 5040), (11760, 5376), (12096, 336), (12096, 672), (12096, 1008), (12096, 1344), (12096, 1680), (12096, 2016), (12096, 2352), (12096, 2688), (12096, 3024), (12096, 3360), (12096, 3696), (12096, 4032), (12096, 4368), (12096, 4704), (12096, 5040), (12096, 5376), (12432, 336), (12432, 672), (12432, 1008), (12432, 1344), (12432, 1680), (12432, 2016), (12432, 2352), (12432, 2688), (12432, 3024), (12432, 3360), (12432, 3696), (12432, 4032), (12432, 4368), (12432, 4704), (12432, 5040), (12768, 336), (12768, 672), (12768, 1008), (12768, 1344), (12768, 1680), (12768, 2016), (12768, 2352), (12768, 2688), (12768, 3024), (12768, 3360), (12768, 3696), (12768, 4032), (12768, 4368), (12768, 4704), (12768, 5040), (13104, 336), (13104, 672), (13104, 1008), (13104, 1344), (13104, 1680), (13104, 2016), (13104, 2352), (13104, 2688), (13104, 3024), (13104, 3360), (13104, 3696), (13104, 4032), (13104, 4368), (13104, 4704), (13440, 336), (13440, 672), (13440, 1008), (13440, 1344), (13440, 1680), (13440, 2016), (13440, 2352), (13440, 2688), (13440, 3024), (13440, 3360), (13440, 3696), (13440, 4032), (13440, 4368), (13440, 4704), (13776, 336), (13776, 672), (13776, 1008), (13776, 1344), (13776, 1680), (13776, 2016), (13776, 2352), (13776, 2688), (13776, 3024), (13776, 3360), (13776, 3696), (13776, 4032), (13776, 4368), (13776, 4704), (14112, 336), (14112, 672), (14112, 1008), (14112, 1344), (14112, 1680), (14112, 2016), (14112, 2352), (14112, 2688), (14112, 3024), (14112, 3360), (14112, 3696), (14112, 4032), (14112, 4368), (14448, 336), (14448, 672), (14448, 1008), (14448, 1344), (14448, 1680), (14448, 2016), (14448, 2352), (14448, 2688), (14448, 3024), (14448, 3360), (14448, 3696), (14448, 4032), (14448, 4368), (14784, 336), (14784, 672), (14784, 1008), (14784, 1344), (14784, 1680), (14784, 2016), (14784, 2352), (14784, 2688), (14784, 3024), (14784, 3360), (14784, 3696), (14784, 4032), (14784, 4368), (15120, 336), (15120, 672), (15120, 1008), (15120, 1344), (15120, 1680), (15120, 2016), (15120, 2352), (15120, 2688), (15120, 3024), (15120, 3360), (15120, 3696), (15120, 4032), (15456, 336), (15456, 672), (15456, 1008), (15456, 1344), (15456, 1680), (15456, 2016), (15456, 2352), (15456, 2688), (15456, 3024), (15456, 3360), (15456, 3696), (15456, 4032), (15792, 336), (15792, 672), (15792, 1008), (15792, 1344), (15792, 1680), (15792, 2016), (15792, 2352), (15792, 2688), (15792, 3024), (15792, 3360), (15792, 3696), (15792, 4032), (16128, 336), (16128, 672), (16128, 1008), (16128, 1344), (16128, 1680), (16128, 2016), (16128, 2352), (16128, 2688), (16128, 3024), (16128, 3360), (16128, 3696), (16128, 4032), (16464, 336), (16464, 672), (16464, 1008), (16464, 1344), (16464, 1680), (16464, 2016), (16464, 2352), (16464, 2688), (16464, 3024), (16464, 3360), (16464, 3696), (16800, 672), (16800, 1008), (16800, 1344), (16800, 1680), (16800, 2016), (16800, 2352), (16800, 2688), (16800, 3024), (16800, 3360), (16800, 3696), (17136, 672), (17136, 1008), (17136, 1344), (17136, 1680), (17136, 2016), (17136, 2352), (17136, 2688), (17136, 3024), (17136, 3360), (17136, 3696), (17472, 672), (17472, 1008), (17472, 1344), (17472, 1680), (17472, 2016), (17472, 2352), (17472, 2688), (17472, 3024), (17472, 3360), (17472, 3696), (17808, 672), (17808, 1008), (17808, 1344), (17808, 1680), (17808, 2016), (17808, 2352), (17808, 2688), (17808, 3024), (17808, 3360), (18144, 672), (18144, 1008), (18144, 1344), (18144, 1680), (18144, 2016), (18144, 2352), (18144, 2688), (18144, 3024), (18144, 3360), (18480, 672), (18480, 1008), (18480, 1344), (18480, 1680), (18480, 2016), (18480, 2352), (18480, 2688), (18480, 3024), (18480, 3360), (18816, 672), (18816, 1008), (18816, 1344), (18816, 1680), (18816, 2016), (18816, 2352), (18816, 2688), (18816, 3024), (18816, 3360), (19152, 672), (19152, 1008), (19152, 1344), (19152, 1680), (19152, 2016), (19152, 2352), (19152, 2688), (19152, 3024), (19152, 3360), (19488, 672), (19488, 1008), (19488, 1344), (19488, 1680), (19488, 2016), (19488, 2352), (19488, 2688), (19488, 3024), (19824, 672), (19824, 1008), (19824, 1344), (19824, 1680), (19824, 2016), (19824, 2352), (19824, 2688), (19824, 3024), (20160, 672), (20160, 1008), (20160, 1344), (20160, 1680), (20160, 2016), (20160, 2352), (20160, 2688), (20160, 3024), (20496, 672), (20496, 1008), (20496, 1344), (20496, 1680), (20496, 2016), (20496, 2352), (20496, 2688), (20496, 3024), (20832, 672), (20832, 1008), (20832, 1344), (20832, 1680), (20832, 2016), (20832, 2352), (20832, 2688), (20832, 3024), (21168, 672), (21168, 1008), (21168, 1344), (21168, 1680), (21168, 2016), (21168, 2352), (21168, 2688), (21168, 3024), (21504, 672), (21504, 1008), (21504, 1344), (21504, 1680), (21504, 2016), (21504, 2352), (21504, 2688), (21504, 3024), (21840, 672), (21840, 1008), (21840, 1344), (21840, 1680), (21840, 2016), (21840, 2352), (21840, 2688), (22176, 672), (22176, 1008), (22176, 1344), (22176, 1680), (22176, 2016), (22176, 2352), (22176, 2688), (22512, 672), (22512, 1008), (22512, 1344), (22512, 1680), (22512, 2016), (22512, 2352), (22512, 2688), (22848, 672), (22848, 1008), (22848, 1344), (22848, 1680), (22848, 2016), (22848, 2352), (22848, 2688), (23184, 672), (23184, 1008), (23184, 1344), (23184, 1680), (23184, 2016), (23184, 2352), (23184, 2688), (23520, 672), (23520, 1008), (23520, 1344), (23520, 1680), (23520, 2016), (23520, 2352), (23520, 2688), (23856, 672), (23856, 1008), (23856, 1344), (23856, 1680), (23856, 2016), (23856, 2352), (23856, 2688), (24192, 672), (24192, 1008), (24192, 1344), (24192, 1680), (24192, 2016), (24192, 2352), (24192, 2688), (24528, 672), (24528, 1008), (24528, 1344), (24528, 1680), (24528, 2016), (24528, 2352), (24864, 672), (24864, 1008), (24864, 1344), (24864, 1680), (24864, 2016), (24864, 2352), (25200, 672), (25200, 1008), (25200, 1344), (25200, 1680), (25200, 2016), (25200, 2352), (25536, 672), (25536, 1008), (25536, 1344), (25536, 1680), (25536, 2016), (25536, 2352), (25872, 672), (25872, 1008), (25872, 1344), (25872, 1680), (25872, 2016), (25872, 2352), (26208, 672), (26208, 1008), (26208, 1344), (26208, 1680), (26208, 2016), (26208, 2352), (26544, 672), (26544, 1008), (26544, 1344), (26544, 1680), (26544, 2016), (26544, 2352), (26880, 672), (26880, 1008), (26880, 1344), (26880, 1680), (26880, 2016), (26880, 2352), (27216, 672), (27216, 1008), (27216, 1344), (27216, 1680), (27216, 2016), (27216, 2352), (27552, 672), (27552, 1008), (27552, 1344), (27552, 1680), (27552, 2016), (27552, 2352), (27888, 672), (27888, 1008), (27888, 1344), (27888, 1680), (27888, 2016), (28224, 672), (28224, 1008), (28224, 1344), (28224, 1680), (28224, 2016), (28560, 672), (28560, 1008), (28560, 1344), (28560, 1680), (28560, 2016), (28896, 672), (28896, 1008), (28896, 1344), (28896, 1680), (28896, 2016), (29232, 672), (29232, 1008), (29232, 1344), (29232, 1680), (29232, 2016), (29568, 672), (29568, 1008), (29568, 1344), (29568, 1680), (29568, 2016), (29904, 672), (29904, 1008), (29904, 1344), (29904, 1680), (29904, 2016), (30240, 672), (30240, 1008), (30240, 1344), (30240, 1680), (30240, 2016), (30576, 672), (30576, 1008), (30576, 1344), (30576, 1680), (30576, 2016), (30912, 672), (30912, 1008), (30912, 1344), (30912, 1680), (30912, 2016), (31248, 672), (31248, 1008), (31248, 1344), (31248, 1680), (31248, 2016), (31584, 672), (31584, 1008), (31584, 1344), (31584, 1680), (31584, 2016), (31920, 672), (31920, 1008), (31920, 1344), (31920, 1680), (31920, 2016), (32256, 672), (32256, 1008), (32256, 1344), (32256, 1680), (32256, 2016), (32592, 672), (32592, 1008), (32592, 1344), (32592, 1680), (32928, 672), (32928, 1008), (32928, 1344), (32928, 1680), (33264, 1008), (33264, 1344), (33264, 1680), (33600, 1008), (33600, 1344), (33600, 1680), (33936, 1008), (33936, 1344), (33936, 1680), (34272, 1008), (34272, 1344), (34272, 1680), (34608, 1008), (34608, 1344), (34608, 1680), (34944, 1008), (34944, 1344), (34944, 1680), (35280, 1008), (35280, 1344), (35280, 1680), (35616, 1008), (35616, 1344), (35616, 1680), (35952, 1008), (35952, 1344), (35952, 1680), (36288, 1008), (36288, 1344), (36288, 1680), (36624, 1008), (36624, 1344), (36624, 1680), (36960, 1008), (36960, 1344), (36960, 1680), (37296, 1008), (37296, 1344), (37296, 1680), (37632, 1008), (37632, 1344), (37632, 1680), (37968, 1008), (37968, 1344), (37968, 1680), (38304, 1008), (38304, 1344), (38304, 1680), (38640, 1008), (38640, 1344), (38640, 1680), (38976, 1008), (38976, 1344), (39312, 1008), (39312, 1344), (39648, 1008), (39648, 1344), (39984, 1008), (39984, 1344), (40320, 1008), (40320, 1344), (40656, 1008), (40656, 1344), (40992, 1008), (40992, 1344), (41328, 1008), (41328, 1344), (41664, 1008), (41664, 1344), (42000, 1008), (42000, 1344), (42336, 1008), (42336, 1344), (42672, 1008), (42672, 1344), (43008, 1008), (43008, 1344), (43344, 1008), (43344, 1344), (43680, 1008), (43680, 1344), (44016, 1008), (44016, 1344), (44352, 1008), (44352, 1344), (44688, 1008), (44688, 1344), (45024, 1008), (45024, 1344), (45360, 1008), (45360, 1344), (45696, 1008), (45696, 1344), (46032, 1008), (46032, 1344), (46368, 1008), (46368, 1344), (46704, 1008), (46704, 1344), (47040, 1008), (47040, 1344), (47376, 1008), (47376, 1344), (47712, 1008), (47712, 1344), (48048, 1008), (48048, 1344), (48384, 1008), (48384, 1344), (48720, 1008), (49056, 1008), (49392, 1008)]" \
    --mm_patch_merge_type unires_max${MAX_TOKENS} \
    --bf16 True \
    --run_name $RUN_NAME \
    --output_dir "./checkpoints/${RUN_NAME}" \
    --num_train_epochs ${NUM_TRAIN_EPOCHS} \
    --per_device_train_batch_size 1 \
    --per_device_eval_batch_size 2 \
    --gradient_accumulation_steps 4 \
    --evaluation_strategy "no" \
    --save_strategy "steps" \
    --save_steps 100 \
    --save_total_limit 3 \
    --learning_rate 5e-6 \
    --weight_decay 0. \
    --warmup_ratio 0.1 \
    --lr_scheduler_type "cosine" \
    --logging_steps 1 \
    --tf32 True \
    --model_max_length 32768 \
    --gradient_checkpointing True \
    --dataloader_num_workers 4 \
    --lazy_preprocess True \
    --report_to tensorboard \
    --torch_compile True \
    --torch_compile_backend "inductor" \
    --dataloader_drop_last True
    # --dataloader_drop_last True \
    # --attn_implementation sdpa

# You can delete the sdpa attn_implementation if you want to use flash attn
