src/constants.py

"""
Constants and mappings for PazaBench.

This module contains all mapping dictionaries and configuration constants
that are shared across the application.
"""

import json
from pathlib import Path

_DATA_DIR = Path(__file__).parent / "data"


def _load_json(name: str):
    """Load a JSON data file bundled under ``src/data/``."""
    with open(_DATA_DIR / name, encoding="utf-8") as data_file:
        return json.load(data_file)


# =============================================================================
# File Paths
# =============================================================================

RESULTS_CSV_PATH = Path("results_summary.csv")
RESULTS_CSV_FILENAME = "results_summary.csv"

MODEL_FAMILY_ALIASES = {
    "hubert": "facebook_hubert",
}


def canonicalize_model_family(model_family: str) -> str:
    """Map legacy or duplicate family identifiers to a single canonical value."""
    return MODEL_FAMILY_ALIASES.get(model_family, model_family)

# =============================================================================
# Filter Configuration
# =============================================================================

FILTER_COLUMN_ORDER = ["model", "language", "dataset_group"]
FILTER_PARAM_MAP = {
    "model": "models",
    "language": "languages",
    "dataset_group": "dataset_groups",
}

# =============================================================================
# Display Configuration
# =============================================================================

ASR_DISPLAY_COLUMNS = [
    "model_family",
    "model",
    "dataset_group",
    "split",
    "language",
    "region",
    "cer",
    "wer",
    "rtfx",
    "duration_sec",
    "inference_time_sec",
    "num_samples",
]
ASR_NUMERIC_COLUMNS = ["wer", "cer", "rtfx", "duration_sec", "inference_time_sec", "num_samples"]
ASR_TEXT_COLUMNS = ["model_family", "model", "dataset_group", "split", "language", "region"]

# =============================================================================
# Metric Configuration
# =============================================================================

METRIC_CONFIGS = {
    "cer": {"label": "CER", "better": "lower", "fmt": "{:.2f}"},
    "wer": {"label": "WER", "better": "lower", "fmt": "{:.2f}"},
    "rtfx": {"label": "RTFx", "better": "higher", "fmt": "{:.2f}"},
}

VIEW_MODE_COLUMNS = {
    "Model families": "model_family",
    "Individual models": "model",
}
DEFAULT_VIEW_MODE = "Model families"

# =============================================================================
# Language Normalization
# =============================================================================

LANGUAGE_NAME_MAPPING = _load_json("language_name_mapping.json")

# =============================================================================
# Geographic Mappings
# =============================================================================

# Language to country mapping for Africa map (using ISO 3166-1 alpha-3 codes)
LANGUAGE_COUNTRY_MAP = _load_json("language_country_map.json")

# Country code to name mapping
COUNTRY_NAMES = _load_json("country_names.json")

# All African countries (ISO 3166-1 alpha-3 -> name), used to show "No coverage"
# on the map for countries without benchmark data.
AFRICAN_COUNTRIES = _load_json("african_countries.json")

# Language to countries mapping (full country names)
# Used for language metadata and region lookups
LANGUAGE_TO_COUNTRIES_MAP: dict[str, list[str]] = _load_json("language_to_countries_map.json")

# Country to African region mapping (geographical)
COUNTRY_TO_REGION_MAP: dict[str, str] = _load_json("country_to_region_map.json")

# =============================================================================
# Visualization Interpretation Text
# =============================================================================

INTERPRETATIONS = {
    'speed_accuracy': """
    - Each bubble represents a specific model; **bubble size = model parameter count**
    - **X-axis (WER)**: Left is better (more accurate)
    - **Y-axis (RTFx)**: Up is better (faster processing)
    - **Top-left quadrant (⭐)**: Ideal zone - fast AND accurate models
    - Gray dashed lines show median values for reference
    - Uses median values to reduce impact of outliers
    - Hover over bubbles to see exact parameter counts (e.g., 1.5B, 300M)
    """,
    
    'leaderboard': """
    - **No language selected**: Shows model families (aggregated across all languages)
    - **Language(s) selected**: Shows top 15 individual models for those languages
    - The horizontal bars show the median Word Error Rate (WER)
    - Lower WER values (left side) indicate better accuracy
    - Error bars represent the standard deviation, showing variability
    - Bar colors correspond to each model family's assigned color
    - Hover over bars to see additional metrics like RTFx (speed) and total samples evaluated
    - Uses median instead of mean to reduce impact of outliers
    """,
    
    'cer_leaderboard': """
    - **No language selected**: Shows model families (aggregated across all languages)
    - **Language(s) selected**: Shows top 15 individual models for those languages
    - The horizontal bars show the median Character Error Rate (CER)
    - Lower CER values (left side) indicate better accuracy
    - CER is especially important for agglutinative and low-resource languages
    - Error bars represent the standard deviation, showing variability
    - Bar colors correspond to each model family's assigned color
    - Hover over bars to see additional metrics like WER, RTFx (speed) and total samples evaluated
    - Uses median instead of mean to reduce impact of outliers
    """,
    
    'correlation': """
    - Each point represents one evaluation result
    - Strong positive correlation means CER and WER move together
    - Models with high character errors typically also have high word errors
    - The trend line shows the overall relationship
    - **Below the line**: Models make more accurate character-level predictions (phonetically closer errors)
    - **Above the line**: Models make more severe character-level errors per word mistake
    - Can be filtered by language to analyze specific language patterns
    """,
    
    'consistency': """
    - Coefficient of Variation (CV) = (Standard Deviation / Median) × 100%
    - **Lower CV** = more consistent performance across different languages
    - **Higher CV** = performance varies widely depending on the language
    - Bar colors correspond to each model family's assigned color
    - Important for production deployment - you want consistent models
    - Outliers have been removed using IQR method for more robust analysis
    - Uses median instead of mean for more robust central tendency measure
    """
}