set -u
	path=$1
set +u
num_processes=1
output=${path}.dp$num_processes
mkdir -p $output
HF_ENDPOINT=https://hf-mirror.com accelerate launch \
  --num_processes=$num_processes \
  run_evals_accelerate.py \
  --override_batch_size 1 \
  --model_args="pretrained=${path},trust_remote_code=True" \
  --custom_tasks "lighteval_tasks_v3.py" \
  --output_dir $output \
  --tasks "custom|hellaswag|0|1,custom|winogrande|0|1,custom|piqa|0|1,custom|siqa|0|1,custom|openbookqa|0|1,custom|arc:easy|0|1,custom|arc:challenge|0|1,custom|commonsense_qa|0|1,custom|trivia_qa|0|1,custom|mmlu_pro_cloze|0|1,custom|gsm8k|5|1,custom|mmlu_cloze:abstract_algebra|0|1,custom|mmlu_cloze:anatomy|0|1,custom|mmlu_cloze:astronomy|0|1,custom|mmlu_cloze:business_ethics|0|1,custom|mmlu_cloze:clinical_knowledge|0|1,custom|mmlu_cloze:college_biology|0|1,custom|mmlu_cloze:college_chemistry|0|1,custom|mmlu_cloze:college_computer_science|0|1,custom|mmlu_cloze:college_mathematics|0|1,custom|mmlu_cloze:college_medicine|0|1,custom|mmlu_cloze:college_physics|0|1,custom|mmlu_cloze:computer_security|0|1,custom|mmlu_cloze:conceptual_physics|0|1,custom|mmlu_cloze:econometrics|0|1,custom|mmlu_cloze:electrical_engineering|0|1,custom|mmlu_cloze:elementary_mathematics|0|1,custom|mmlu_cloze:formal_logic|0|1,custom|mmlu_cloze:global_facts|0|1,custom|mmlu_cloze:high_school_biology|0|1,custom|mmlu_cloze:high_school_chemistry|0|1,custom|mmlu_cloze:high_school_computer_science|0|1,custom|mmlu_cloze:high_school_european_history|0|1,custom|mmlu_cloze:high_school_geography|0|1,custom|mmlu_cloze:high_school_government_and_politics|0|1,custom|mmlu_cloze:high_school_macroeconomics|0|1,custom|mmlu_cloze:high_school_mathematics|0|1,custom|mmlu_cloze:high_school_microeconomics|0|1,custom|mmlu_cloze:high_school_physics|0|1,custom|mmlu_cloze:high_school_psychology|0|1,custom|mmlu_cloze:high_school_statistics|0|1,custom|mmlu_cloze:high_school_us_history|0|1,custom|mmlu_cloze:high_school_world_history|0|1,custom|mmlu_cloze:human_aging|0|1,custom|mmlu_cloze:human_sexuality|0|1,custom|mmlu_cloze:international_law|0|1,custom|mmlu_cloze:jurisprudence|0|1,custom|mmlu_cloze:logical_fallacies|0|1,custom|mmlu_cloze:machine_learning|0|1,custom|mmlu_cloze:management|0|1,custom|mmlu_cloze:marketing|0|1,custom|mmlu_cloze:medical_genetics|0|1,custom|mmlu_cloze:miscellaneous|0|1,custom|mmlu_cloze:moral_disputes|0|1,custom|mmlu_cloze:moral_scenarios|0|1,custom|mmlu_cloze:nutrition|0|1,custom|mmlu_cloze:philosophy|0|1,custom|mmlu_cloze:prehistory|0|1,custom|mmlu_cloze:professional_accounting|0|1,custom|mmlu_cloze:professional_law|0|1,custom|mmlu_cloze:professional_medicine|0|1,custom|mmlu_cloze:professional_psychology|0|1,custom|mmlu_cloze:public_relations|0|1,custom|mmlu_cloze:security_studies|0|1,custom|mmlu_cloze:sociology|0|1,custom|mmlu_cloze:us_foreign_policy|0|1,custom|mmlu_cloze:virology|0|1,custom|mmlu_cloze:world_religions|0|1,custom|cmmlu:agronomy|0|1,custom|cmmlu:anatomy|0|1,custom|cmmlu:ancient_chinese|0|1,custom|cmmlu:arts|0|1,custom|cmmlu:astronomy|0|1,custom|cmmlu:business_ethics|0|1,custom|cmmlu:chinese_civil_service_exam|0|1,custom|cmmlu:chinese_driving_rule|0|1,custom|cmmlu:chinese_food_culture|0|1,custom|cmmlu:chinese_foreign_policy|0|1,custom|cmmlu:chinese_history|0|1,custom|cmmlu:chinese_literature|0|1,custom|cmmlu:chinese_teacher_qualification|0|1,custom|cmmlu:clinical_knowledge|0|1,custom|cmmlu:college_actuarial_science|0|1,custom|cmmlu:college_education|0|1,custom|cmmlu:college_engineering_hydrology|0|1,custom|cmmlu:college_law|0|1,custom|cmmlu:college_mathematics|0|1,custom|cmmlu:college_medical_statistics|0|1,custom|cmmlu:college_medicine|0|1,custom|cmmlu:computer_science|0|1,custom|cmmlu:computer_security|0|1,custom|cmmlu:conceptual_physics|0|1,custom|cmmlu:construction_project_management|0|1,custom|cmmlu:economics|0|1,custom|cmmlu:education|0|1,custom|cmmlu:electrical_engineering|0|1,custom|cmmlu:elementary_chinese|0|1,custom|cmmlu:elementary_commonsense|0|1,custom|cmmlu:elementary_information_and_technology|0|1,custom|cmmlu:elementary_mathematics|0|1,custom|cmmlu:ethnology|0|1,custom|cmmlu:food_science|0|1,custom|cmmlu:genetics|0|1,custom|cmmlu:global_facts|0|1,custom|cmmlu:high_school_biology|0|1,custom|cmmlu:high_school_chemistry|0|1,custom|cmmlu:high_school_geography|0|1,custom|cmmlu:high_school_mathematics|0|1,custom|cmmlu:high_school_physics|0|1,custom|cmmlu:high_school_politics|0|1,custom|cmmlu:human_sexuality|0|1,custom|cmmlu:international_law|0|1,custom|cmmlu:journalism|0|1,custom|cmmlu:jurisprudence|0|1,custom|cmmlu:legal_and_moral_basis|0|1,custom|cmmlu:logical|0|1,custom|cmmlu:machine_learning|0|1,custom|cmmlu:management|0|1,custom|cmmlu:marketing|0|1,custom|cmmlu:marxist_theory|0|1,custom|cmmlu:modern_chinese|0|1,custom|cmmlu:nutrition|0|1,custom|cmmlu:philosophy|0|1,custom|cmmlu:professional_accounting|0|1,custom|cmmlu:professional_law|0|1,custom|cmmlu:professional_medicine|0|1,custom|cmmlu:professional_psychology|0|1,custom|cmmlu:public_relations|0|1,custom|cmmlu:security_study|0|1,custom|cmmlu:sociology|0|1,custom|cmmlu:sports_science|0|1,custom|cmmlu:traditional_chinese_medicine|0|1,custom|cmmlu:virology|0|1,custom|cmmlu:world_history|0|1,custom|cmmlu:world_religions|0|1,custom|ceval:computer_network|0|1,custom|ceval:operating_system|0|1,custom|ceval:computer_architecture|0|1,custom|ceval:college_programming|0|1,custom|ceval:college_physics|0|1,custom|ceval:college_chemistry|0|1,custom|ceval:advanced_mathematics|0|1,custom|ceval:probability_and_statistics|0|1,custom|ceval:discrete_mathematics|0|1,custom|ceval:electrical_engineer|0|1,custom|ceval:metrology_engineer|0|1,custom|ceval:high_school_mathematics|0|1,custom|ceval:high_school_physics|0|1,custom|ceval:high_school_chemistry|0|1,custom|ceval:high_school_biology|0|1,custom|ceval:middle_school_mathematics|0|1,custom|ceval:middle_school_biology|0|1,custom|ceval:middle_school_physics|0|1,custom|ceval:middle_school_chemistry|0|1,custom|ceval:veterinary_medicine|0|1,custom|ceval:college_economics|0|1,custom|ceval:business_administration|0|1,custom|ceval:marxism|0|1,custom|ceval:mao_zedong_thought|0|1,custom|ceval:education_science|0|1,custom|ceval:teacher_qualification|0|1,custom|ceval:high_school_politics|0|1,custom|ceval:high_school_geography|0|1,custom|ceval:middle_school_politics|0|1,custom|ceval:middle_school_geography|0|1,custom|ceval:modern_chinese_history|0|1,custom|ceval:ideological_and_moral_cultivation|0|1,custom|ceval:logic|0|1,custom|ceval:law|0|1,custom|ceval:chinese_language_and_literature|0|1,custom|ceval:art_studies|0|1,custom|ceval:professional_tour_guide|0|1,custom|ceval:legal_professional|0|1,custom|ceval:high_school_chinese|0|1,custom|ceval:high_school_history|0|1,custom|ceval:middle_school_history|0|1,custom|ceval:civil_servant|0|1,custom|ceval:sports_science|0|1,custom|ceval:plant_protection|0|1,custom|ceval:basic_medicine|0|1,custom|ceval:clinical_medicine|0|1,custom|ceval:urban_and_rural_planner|0|1,custom|ceval:accountant|0|1,custom|ceval:fire_engineer|0|1,custom|ceval:environmental_impact_assessment_engineer|0|1,custom|ceval:tax_accountant|0|1,custom|ceval:physician|0|1"
#  --tasks "custom|gsm8k|5|1"
#  --tasks "custom|hellaswag|0|1,custom|winogrande|0|1,custom|piqa|0|1,custom|siqa|0|1,custom|openbookqa|0|1,custom|arc:easy|0|1,custom|arc:challenge|0|1,custom|commonsense_qa|0|1,custom|trivia_qa|0|1,custom|mmlu_pro_cloze|0|1,custom|mmlu_cloze:abstract_algebra|0|1,custom|mmlu_cloze:anatomy|0|1,custom|mmlu_cloze:astronomy|0|1,custom|mmlu_cloze:business_ethics|0|1,custom|mmlu_cloze:clinical_knowledge|0|1,custom|mmlu_cloze:college_biology|0|1,custom|mmlu_cloze:college_chemistry|0|1,custom|mmlu_cloze:college_computer_science|0|1,custom|mmlu_cloze:college_mathematics|0|1,custom|mmlu_cloze:college_medicine|0|1,custom|mmlu_cloze:college_physics|0|1,custom|mmlu_cloze:computer_security|0|1,custom|mmlu_cloze:conceptual_physics|0|1,custom|mmlu_cloze:econometrics|0|1,custom|mmlu_cloze:electrical_engineering|0|1,custom|mmlu_cloze:elementary_mathematics|0|1,custom|mmlu_cloze:formal_logic|0|1,custom|mmlu_cloze:global_facts|0|1,custom|mmlu_cloze:high_school_biology|0|1,custom|mmlu_cloze:high_school_chemistry|0|1,custom|mmlu_cloze:high_school_computer_science|0|1,custom|mmlu_cloze:high_school_european_history|0|1,custom|mmlu_cloze:high_school_geography|0|1,custom|mmlu_cloze:high_school_government_and_politics|0|1,custom|mmlu_cloze:high_school_macroeconomics|0|1,custom|mmlu_cloze:high_school_mathematics|0|1,custom|mmlu_cloze:high_school_microeconomics|0|1,custom|mmlu_cloze:high_school_physics|0|1,custom|mmlu_cloze:high_school_psychology|0|1,custom|mmlu_cloze:high_school_statistics|0|1,custom|mmlu_cloze:high_school_us_history|0|1,custom|mmlu_cloze:high_school_world_history|0|1,custom|mmlu_cloze:human_aging|0|1,custom|mmlu_cloze:human_sexuality|0|1,custom|mmlu_cloze:international_law|0|1,custom|mmlu_cloze:jurisprudence|0|1,custom|mmlu_cloze:logical_fallacies|0|1,custom|mmlu_cloze:machine_learning|0|1,custom|mmlu_cloze:management|0|1,custom|mmlu_cloze:marketing|0|1,custom|mmlu_cloze:medical_genetics|0|1,custom|mmlu_cloze:miscellaneous|0|1,custom|mmlu_cloze:moral_disputes|0|1,custom|mmlu_cloze:moral_scenarios|0|1,custom|mmlu_cloze:nutrition|0|1,custom|mmlu_cloze:philosophy|0|1,custom|mmlu_cloze:prehistory|0|1,custom|mmlu_cloze:professional_accounting|0|1,custom|mmlu_cloze:professional_law|0|1,custom|mmlu_cloze:professional_medicine|0|1,custom|mmlu_cloze:professional_psychology|0|1,custom|mmlu_cloze:public_relations|0|1,custom|mmlu_cloze:security_studies|0|1,custom|mmlu_cloze:sociology|0|1,custom|mmlu_cloze:us_foreign_policy|0|1,custom|mmlu_cloze:virology|0|1,custom|mmlu_cloze:world_religions|0|1,custom|cmmlu:agronomy|0|1,custom|cmmlu:anatomy|0|1,custom|cmmlu:ancient_chinese|0|1,custom|cmmlu:arts|0|1,custom|cmmlu:astronomy|0|1,custom|cmmlu:business_ethics|0|1,custom|cmmlu:chinese_civil_service_exam|0|1,custom|cmmlu:chinese_driving_rule|0|1,custom|cmmlu:chinese_food_culture|0|1,custom|cmmlu:chinese_foreign_policy|0|1,custom|cmmlu:chinese_history|0|1,custom|cmmlu:chinese_literature|0|1,custom|cmmlu:chinese_teacher_qualification|0|1,custom|cmmlu:clinical_knowledge|0|1,custom|cmmlu:college_actuarial_science|0|1,custom|cmmlu:college_education|0|1,custom|cmmlu:college_engineering_hydrology|0|1,custom|cmmlu:college_law|0|1,custom|cmmlu:college_mathematics|0|1,custom|cmmlu:college_medical_statistics|0|1,custom|cmmlu:college_medicine|0|1,custom|cmmlu:computer_science|0|1,custom|cmmlu:computer_security|0|1,custom|cmmlu:conceptual_physics|0|1,custom|cmmlu:construction_project_management|0|1,custom|cmmlu:economics|0|1,custom|cmmlu:education|0|1,custom|cmmlu:electrical_engineering|0|1,custom|cmmlu:elementary_chinese|0|1,custom|cmmlu:elementary_commonsense|0|1,custom|cmmlu:elementary_information_and_technology|0|1,custom|cmmlu:elementary_mathematics|0|1,custom|cmmlu:ethnology|0|1,custom|cmmlu:food_science|0|1,custom|cmmlu:genetics|0|1,custom|cmmlu:global_facts|0|1,custom|cmmlu:high_school_biology|0|1,custom|cmmlu:high_school_chemistry|0|1,custom|cmmlu:high_school_geography|0|1,custom|cmmlu:high_school_mathematics|0|1,custom|cmmlu:high_school_physics|0|1,custom|cmmlu:high_school_politics|0|1,custom|cmmlu:human_sexuality|0|1,custom|cmmlu:international_law|0|1,custom|cmmlu:journalism|0|1,custom|cmmlu:jurisprudence|0|1,custom|cmmlu:legal_and_moral_basis|0|1,custom|cmmlu:logical|0|1,custom|cmmlu:machine_learning|0|1,custom|cmmlu:management|0|1,custom|cmmlu:marketing|0|1,custom|cmmlu:marxist_theory|0|1,custom|cmmlu:modern_chinese|0|1,custom|cmmlu:nutrition|0|1,custom|cmmlu:philosophy|0|1,custom|cmmlu:professional_accounting|0|1,custom|cmmlu:professional_law|0|1,custom|cmmlu:professional_medicine|0|1,custom|cmmlu:professional_psychology|0|1,custom|cmmlu:public_relations|0|1,custom|cmmlu:security_study|0|1,custom|cmmlu:sociology|0|1,custom|cmmlu:sports_science|0|1,custom|cmmlu:traditional_chinese_medicine|0|1,custom|cmmlu:virology|0|1,custom|cmmlu:world_history|0|1,custom|cmmlu:world_religions|0|1,custom|ceval:computer_network|0|1,custom|ceval:operating_system|0|1,custom|ceval:computer_architecture|0|1,custom|ceval:college_programming|0|1,custom|ceval:college_physics|0|1,custom|ceval:college_chemistry|0|1,custom|ceval:advanced_mathematics|0|1,custom|ceval:probability_and_statistics|0|1,custom|ceval:discrete_mathematics|0|1,custom|ceval:electrical_engineer|0|1,custom|ceval:metrology_engineer|0|1,custom|ceval:high_school_mathematics|0|1,custom|ceval:high_school_physics|0|1,custom|ceval:high_school_chemistry|0|1,custom|ceval:high_school_biology|0|1,custom|ceval:middle_school_mathematics|0|1,custom|ceval:middle_school_biology|0|1,custom|ceval:middle_school_physics|0|1,custom|ceval:middle_school_chemistry|0|1,custom|ceval:veterinary_medicine|0|1,custom|ceval:college_economics|0|1,custom|ceval:business_administration|0|1,custom|ceval:marxism|0|1,custom|ceval:mao_zedong_thought|0|1,custom|ceval:education_science|0|1,custom|ceval:teacher_qualification|0|1,custom|ceval:high_school_politics|0|1,custom|ceval:high_school_geography|0|1,custom|ceval:middle_school_politics|0|1,custom|ceval:middle_school_geography|0|1,custom|ceval:modern_chinese_history|0|1,custom|ceval:ideological_and_moral_cultivation|0|1,custom|ceval:logic|0|1,custom|ceval:law|0|1,custom|ceval:chinese_language_and_literature|0|1,custom|ceval:art_studies|0|1,custom|ceval:professional_tour_guide|0|1,custom|ceval:legal_professional|0|1,custom|ceval:high_school_chinese|0|1,custom|ceval:high_school_history|0|1,custom|ceval:middle_school_history|0|1,custom|ceval:civil_servant|0|1,custom|ceval:sports_science|0|1,custom|ceval:plant_protection|0|1,custom|ceval:basic_medicine|0|1,custom|ceval:clinical_medicine|0|1,custom|ceval:urban_and_rural_planner|0|1,custom|ceval:accountant|0|1,custom|ceval:fire_engineer|0|1,custom|ceval:environmental_impact_assessment_engineer|0|1,custom|ceval:tax_accountant|0|1,custom|ceval:physician|0|1"
#   --multi_gpu \ for multiple gpus
