#!/bin/bash

# DeepSeek model version comparison tool testing script for OpenRouter
# Usage: ./test_deepseek_versions.sh YOUR_API_KEY [quick|full]

if [ $# -lt 1 ]; then
    echo "Usage: ./test_deepseek_versions.sh YOUR_OPENROUTER_API_KEY [quick|full]"
    echo ""
    echo "This script compares three DeepSeek model versions:"
    echo "  - deepseek/deepseek-chat-v3.1"
    echo "  - deepseek/deepseek-chat-v3-0324"
    echo "  - deepseek/deepseek-chat-v3.1:thinking"
    echo ""
    echo "Examples:"
    echo "  ./test_deepseek_versions.sh sk-or-xxx quick"
    echo "  ./test_deepseek_versions.sh sk-or-xxx full"
    exit 1
fi

API_KEY=$1
TEST_MODE=${2:-quick}  # Default to quick mode if not specified

# Create results directory with timestamp
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
RESULTS_DIR="deepseek_comparison_${TIMESTAMP}"
mkdir -p "$RESULTS_DIR"

echo "DeepSeek Model Comparison Configuration:"
echo "  Test Mode: $TEST_MODE"
echo "  Results Directory: $RESULTS_DIR"
echo ""

# DeepSeek models to compare
models=(
    "deepseek/deepseek-chat-v3.1"
    "deepseek/deepseek-chat-v3-0324"
    "deepseek/deepseek-chat-v3.1:thinking"
)

# Function to run test and capture results
run_test() {
    local model=$1
    local safe_model_name=$(echo "$model" | sed 's/[\/:]/_/g')
    local output_file="$RESULTS_DIR/test_${safe_model_name}.txt"
    
    echo "================================================"
    echo "Testing model: $model"
    echo "Output file: $output_file"
    echo "================================================"
    
    # Build the command
    cmd="python3 tool_tester_v2.py \
        --api-base https://openrouter.ai/api/v1 \
        --api-key $API_KEY \
        --model \"$model\" \
        --temperature 0.0"
    
    # Add test mode flag
    if [ "$TEST_MODE" == "quick" ]; then
        cmd="$cmd --quick"
    else
        cmd="$cmd --max-tools 40"
    fi
    
    # Add output file
    cmd="$cmd --output \"$output_file\""
    
    # Run the test and capture both stdout and the result
    echo "Running: $cmd"
    eval $cmd 2>&1 | tee "${output_file}.log"
    
    # Check if the test completed successfully
    if [ ${PIPESTATUS[0]} -eq 0 ]; then
        echo "PASS: Test completed successfully for $model"
        
        # Extract the overall score from the output file if it exists
        if [ -f "$output_file" ]; then
            score=$(grep "OVERALL SCORE:" "$output_file" | tail -1)
            echo "  $score"
        fi
    else
        echo "FAIL: Test failed for $model"
        echo "FAILED: $model" >> "$RESULTS_DIR/failed_models.txt"
    fi
    
    echo ""
    sleep 2  # Small delay between models to avoid rate limiting
}

# Test each model
successful_tests=0
failed_tests=0

for model in "${models[@]}"; do
    run_test "$model"
    
    # Check if test was successful
    if [ $? -eq 0 ]; then
        ((successful_tests++))
    else
        ((failed_tests++))
    fi
done

# Generate summary report
summary_file="$RESULTS_DIR/summary.txt"
echo "================================================" | tee "$summary_file"
echo "DEEPSEEK MODEL COMPARISON SUMMARY REPORT" | tee -a "$summary_file"
echo "================================================" | tee -a "$summary_file"
echo "Test Mode: $TEST_MODE" | tee -a "$summary_file"
echo "Timestamp: $TIMESTAMP" | tee -a "$summary_file"
echo "Total Models Tested: ${#models[@]}" | tee -a "$summary_file"
echo "Successful Tests: $successful_tests" | tee -a "$summary_file"
echo "Failed Tests: $failed_tests" | tee -a "$summary_file"
echo "" | tee -a "$summary_file"

# Extract scores from all successful tests
echo "Model Performance Comparison:" | tee -a "$summary_file"
echo "-----------------------------------------" | tee -a "$summary_file"

for model in "${models[@]}"; do
    safe_model_name=$(echo "$model" | sed 's/[\/:]/_/g')
    file="$RESULTS_DIR/test_${safe_model_name}.txt"
    
    if [ -f "$file" ]; then
        score=$(grep "OVERALL SCORE:" "$file" | tail -1 | awk '{print $3, $4}')
        if [ -n "$score" ]; then
            printf "%-40s %s\n" "$model:" "$score" | tee -a "$summary_file"
        else
            printf "%-40s %s\n" "$model:" "No score available" | tee -a "$summary_file"
        fi
    else
        printf "%-40s %s\n" "$model:" "Test failed" | tee -a "$summary_file"
    fi
done

echo "" | tee -a "$summary_file"
echo "Full results saved in: $RESULTS_DIR" | tee -a "$summary_file"

# Create a CSV summary for easy analysis
csv_file="$RESULTS_DIR/model_comparison.csv"
echo "Model,OverallScore,Grade,ScenarioSuccessRate,ToolPrecision,ToolRecall,ToolF1,ParamAccuracy,ExecutionSuccess" > "$csv_file"

for model in "${models[@]}"; do
    safe_model_name=$(echo "$model" | sed 's/[\/:]/_/g')
    file="$RESULTS_DIR/test_${safe_model_name}.txt"
    
    if [ -f "$file" ]; then
        # Extract metrics using grep and awk
        overall_score=$(grep "OVERALL SCORE:" "$file" | tail -1 | awk '{print $3}' | sed 's/%//')
        grade=$(grep "OVERALL SCORE:" "$file" | tail -1 | awk '{print $4}' | sed 's/[()]//g')
        scenario_rate=$(grep "Scenario Success Rate:" "$file" | tail -1 | awk '{print $4}' | sed 's/%//')
        tool_precision=$(grep "Tool Precision (LLM only):" "$file" | tail -1 | awk '{print $5}' | sed 's/%//')
        tool_recall=$(grep "Tool Recall (LLM only):" "$file" | tail -1 | awk '{print $5}' | sed 's/%//')
        tool_f1=$(grep "Tool F1 (LLM only):" "$file" | tail -1 | awk '{print $5}' | sed 's/%//')
        param_accuracy=$(grep "Parameter Accuracy (structural):" "$file" | tail -1 | awk '{print $4}' | sed 's/%//')
        exec_success=$(grep "Execution Success Rate:" "$file" | tail -1 | awk '{print $4}' | sed 's/%//')
        
        if [ -n "$overall_score" ]; then
            echo "$model,$overall_score,$grade,$scenario_rate,$tool_precision,$tool_recall,$tool_f1,$param_accuracy,$exec_success" >> "$csv_file"
        fi
    fi
done

echo "CSV results saved in: $csv_file"
echo ""
echo "================================================"
echo "DeepSeek Model Comparison Complete!"
echo "================================================"
echo ""
echo "Quick Analysis:"
echo "View the summary with: cat $RESULTS_DIR/summary.txt"
echo "View CSV data with: cat $RESULTS_DIR/model_comparison.csv"
echo ""
echo "For detailed analysis of individual models, check:"
for model in "${models[@]}"; do
    safe_model_name=$(echo "$model" | sed 's/[\/:]/_/g')
    echo "  $model: $RESULTS_DIR/test_${safe_model_name}.txt"
done