#!/bin/bash

# Qwen3 Coder tool testing provider test script for OpenRouter
# Usage: ./test_qwen_coder_providers.sh YOUR_API_KEY [quick|full]

if [ $# -lt 1 ]; then
    echo "Usage: ./test_qwen_coder_providers.sh YOUR_OPENROUTER_API_KEY [quick|full]"
    echo ""
    echo "Examples:"
    echo "  ./test_qwen_coder_providers.sh sk-or-xxx quick"
    echo "  ./test_qwen_coder_providers.sh sk-or-xxx full"
    exit 1
fi

API_KEY=$1
MODEL="qwen/qwen3-coder"
TEST_MODE=${2:-quick}  # Default to quick mode if not specified

# Create results directory with timestamp
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
RESULTS_DIR="tool_test_results_${TIMESTAMP}"
mkdir -p "$RESULTS_DIR"

echo "Tool Testing Configuration:"
echo "  Model: $MODEL"
echo "  Test Mode: $TEST_MODE"
echo "  Results Directory: $RESULTS_DIR"
echo ""

# Providers to test for Qwen3 Coder
providers=(
    "chutes/fp8"
    "deepinfra/fp4"
    "baseten/fp8"
    "parasail/fp8"
    "fireworks"
    "novita/fp8"
    "atlas-cloud/fp8"
    "phala"
    "gmicloud/fp8"
    "targon/fp8"
    "alibaba/opensource"
    "together/fp8"
    "hyperbolic/fp8"
    "cerebras/fp8"
)

# Function to run test and capture results
run_test() {
    local provider=$1
    local output_file="$RESULTS_DIR/test_${provider//\//_}.txt"
    
    echo "================================================"
    echo "Testing provider: $provider"
    echo "Output file: $output_file"
    echo "================================================"
    
    # Build the command
    cmd="python3 tool_tester_v2.py \
        --api-base https://openrouter.ai/api/v1 \
        --api-key $API_KEY \
        --model $MODEL \
        --provider \"$provider\" \
        --temperature 0.6"
    
    # Add test mode flag
    if [ "$TEST_MODE" == "quick" ]; then
        cmd="$cmd --quick"
    else
        cmd="$cmd --max-tools 40"
    fi
    
    # Add output file
    cmd="$cmd --output \"$output_file\""
    
    # Run the test and capture both stdout and the result
    echo "Running: $cmd"
    eval $cmd 2>&1 | tee "${output_file}.log"
    
    # Check if the test completed successfully
    if [ ${PIPESTATUS[0]} -eq 0 ]; then
        echo "PASS: Test completed successfully for $provider"
        
        # Extract the overall score from the output file if it exists
        if [ -f "$output_file" ]; then
            score=$(grep "OVERALL SCORE:" "$output_file" | tail -1)
            echo "  $score"
        fi
    else
        echo "FAIL: Test failed for $provider"
        echo "FAILED: $provider" >> "$RESULTS_DIR/failed_providers.txt"
    fi
    
    echo ""
    sleep 2  # Small delay between providers to avoid rate limiting
}

# Test each provider
successful_tests=0
failed_tests=0

for provider in "${providers[@]}"; do
    # Skip provider if it doesn't make sense for the model
    # (you can add logic here to filter providers based on model)
    
    run_test "$provider"
    
    # Check if test was successful
    if [ $? -eq 0 ]; then
        ((successful_tests++))
    else
        ((failed_tests++))
    fi
done

# Generate summary report
summary_file="$RESULTS_DIR/summary.txt"
echo "================================================" | tee "$summary_file"
echo "TOOL TESTING SUMMARY REPORT" | tee -a "$summary_file"
echo "================================================" | tee -a "$summary_file"
echo "Model: $MODEL" | tee -a "$summary_file"
echo "Test Mode: $TEST_MODE" | tee -a "$summary_file"
echo "Timestamp: $TIMESTAMP" | tee -a "$summary_file"
echo "Total Providers Tested: ${#providers[@]}" | tee -a "$summary_file"
echo "Successful Tests: $successful_tests" | tee -a "$summary_file"
echo "Failed Tests: $failed_tests" | tee -a "$summary_file"
echo "" | tee -a "$summary_file"

# Extract scores from all successful tests
echo "Provider Scores:" | tee -a "$summary_file"
echo "-----------------------------------------" | tee -a "$summary_file"

for file in "$RESULTS_DIR"/test_*.txt; do
    if [ -f "$file" ]; then
        provider_name=$(basename "$file" .txt | sed 's/test_//' | sed 's/_/\//g')
        score=$(grep "OVERALL SCORE:" "$file" | tail -1 | awk '{print $3, $4}')
        if [ -n "$score" ]; then
            printf "%-30s %s\n" "$provider_name:" "$score" | tee -a "$summary_file"
        fi
    fi
done

echo "" | tee -a "$summary_file"
echo "Full results saved in: $RESULTS_DIR" | tee -a "$summary_file"

# Create a CSV summary for easy analysis
csv_file="$RESULTS_DIR/results.csv"
echo "Provider,Model,OverallScore,Grade,ScenarioSuccessRate,ToolPrecision,ToolRecall,ToolF1,ParamAccuracy,ExecutionSuccess" > "$csv_file"

for file in "$RESULTS_DIR"/test_*.txt; do
    if [ -f "$file" ]; then
        provider_name=$(basename "$file" .txt | sed 's/test_//' | sed 's/_/\//g')
        
        # Extract metrics using grep and awk
        overall_score=$(grep "OVERALL SCORE:" "$file" | tail -1 | awk '{print $3}' | sed 's/%//')
        grade=$(grep "OVERALL SCORE:" "$file" | tail -1 | awk '{print $4}' | sed 's/[()]//g')
        scenario_rate=$(grep "Scenario Success Rate:" "$file" | tail -1 | awk '{print $4}' | sed 's/%//')
        tool_precision=$(grep "Tool Precision (LLM only):" "$file" | tail -1 | awk '{print $5}' | sed 's/%//')
        tool_recall=$(grep "Tool Recall (LLM only):" "$file" | tail -1 | awk '{print $5}' | sed 's/%//')
        tool_f1=$(grep "Tool F1 (LLM only):" "$file" | tail -1 | awk '{print $5}' | sed 's/%//')
        param_accuracy=$(grep "Parameter Accuracy (structural):" "$file" | tail -1 | awk '{print $4}' | sed 's/%//')
        exec_success=$(grep "Execution Success Rate:" "$file" | tail -1 | awk '{print $4}' | sed 's/%//')
        
        if [ -n "$overall_score" ]; then
            echo "$provider_name,$MODEL,$overall_score,$grade,$scenario_rate,$tool_precision,$tool_recall,$tool_f1,$param_accuracy,$exec_success" >> "$csv_file"
        fi
    fi
done

echo "CSV results saved in: $csv_file"
echo ""
echo "All providers tested!"