#!/bin/bash

set -e

export CUDA_VISIBLE_DEVICES='0,1,2,3,4,5,6,7'

# add more of models neccessary
models=("gpt-4o-2024-05-13" "gpt-3.5-turbo-0125")
open_models=("llama2-chat-7b" "llama2-chat-13b" "llama2-chat-70b" "llama3-8b-instruct" "llama3-70b-instruct" "Mistral-7B-Instruct-v0.2" "Qwen1.5-14B-Chat")
open_model_paths=("")  # specify the model paths here for open-source models that don't using api

for model in "${models[@]}"; do
  echo "Running inference for model: $model"
  
  python run_demon_gpt.py \
    --model "$model" \
    --output_filename "../../Demonstration_Format/output/${model}/cot/responses.jsonl" \
    --cot
  
  python run_demon_gpt.py \
    --model "$model" \
    --output_filename "../../Demonstration_Format/output/${model}/cot/responses_all_a.jsonl" \
    --cot \
    --all_a
  
  python run_demon_gpt.py \
    --model "$model" \
    --output_filename "../../Demonstration_Format/output/${model}/wo_cot/responses.jsonl"
  
  python run_demon_gpt.py \
    --model "$model" \
    --output_filename "../../Demonstration_Format/output/${model}/wo_cot/responses_all_a.jsonl" \
    --all_a
done

echo "Inference done for all gpt models."

for i in "${!open_models[@]}"; do
  model_name=${open_models[$i]}
  model_path=${open_model_paths[$i]}
  
  echo "Running inference for model: $model_name"

  python run_demon_vllm.py \
    --model_path "$model_path" \
    --output_filename "../../Demonstration_Format/output/${model_name}/cot/responses.jsonl" \
    --cot

  python run_demon_vllm.py \
    --model_path "$model_path" \
    --output_filename "../../Demonstration_Format/output/${model_name}/cot/responses_all_a.jsonl" \
    --cot \
    --all_a

  python run_demon_vllm.py \
    --model_path "$model_path" \
    --output_filename "../../Demonstration_Format/output/${model_name}/wo_cot/responses.jsonl"

  python run_demon_vllm.py \
    --model_path "$model_path" \
    --output_filename "../../Demonstration_Format/output/${model_name}/wo_cot/responses_all_a.jsonl" \
    --all_a

done

echo "Inference done for all open-source models."

for model in "${models[@]}"; do
  echo "Running evaluation for model: $model"
  
  python demon_evaluate.py \
    --model "$model" \
    --input_filename "../../Demonstration_Format/output/${model}/cot/responses.jsonl" \
    --cot \
    --gpt
  
  python demon_evaluate.py \
    --model "$model" \
    --input_filename "../../Demonstration_Format/output/${model}/cot/responses_all_a.jsonl" \
    --cot \
    --gpt
  
  python demon_evaluate.py \
    --model "$model" \
    --input_filename "../../Demonstration_Format/output/${model}/wo_cot/responses.jsonl" \
    --gpt
  
  python demon_evaluate.py \
    --model "$model" \
    --input_filename "../../Demonstration_Format/output/${model}/wo_cot/responses_all_a.jsonl" \
    --gpt
done

echo "Evaluation done for all gpt models."

for i in "${!open_models[@]}"; do
  model_name=${open_models[$i]}
  model_path=${open_model_paths[$i]}
  
  echo "Running evaluation for model: $model_name"

  python demon_evaluate.py \
    --input_filename "../../Demonstration_Format/output/${model_name}/cot/responses.jsonl" \
    --cot

  python demon_evaluate.py \
    --input_filename "../../Demonstration_Format/output/${model_name}/cot/responses_all_a.jsonl" \
    --cot

  python demon_evaluate.py \
    --input_filename "../../Demonstration_Format/output/${model_name}/wo_cot/responses.jsonl"

  python demon_evaluate.py \
    --input_filename "../../Demonstration_Format/output/${model_name}/wo_cot/responses_all_a.jsonl"

done

echo "Evaluation done for all open-source models."