""" Constants and mappings for PazaBench. This module contains all mapping dictionaries and configuration constants that are shared across the application. """ import json from pathlib import Path _DATA_DIR = Path(__file__).parent / "data" def _load_json(name: str): """Load a JSON data file bundled under ``src/data/``.""" with open(_DATA_DIR / name, encoding="utf-8") as data_file: return json.load(data_file) # ============================================================================= # File Paths # ============================================================================= RESULTS_CSV_PATH = Path("results_summary.csv") RESULTS_CSV_FILENAME = "results_summary.csv" MODEL_FAMILY_ALIASES = { "hubert": "facebook_hubert", } def canonicalize_model_family(model_family: str) -> str: """Map legacy or duplicate family identifiers to a single canonical value.""" return MODEL_FAMILY_ALIASES.get(model_family, model_family) # ============================================================================= # Filter Configuration # ============================================================================= FILTER_COLUMN_ORDER = ["model", "language", "dataset_group"] FILTER_PARAM_MAP = { "model": "models", "language": "languages", "dataset_group": "dataset_groups", } # ============================================================================= # Display Configuration # ============================================================================= ASR_DISPLAY_COLUMNS = [ "model_family", "model", "dataset_group", "split", "language", "region", "cer", "wer", "rtfx", "duration_sec", "inference_time_sec", "num_samples", ] ASR_NUMERIC_COLUMNS = ["wer", "cer", "rtfx", "duration_sec", "inference_time_sec", "num_samples"] ASR_TEXT_COLUMNS = ["model_family", "model", "dataset_group", "split", "language", "region"] # ============================================================================= # Metric Configuration # ============================================================================= METRIC_CONFIGS = { "cer": {"label": "CER", "better": "lower", "fmt": "{:.2f}"}, "wer": {"label": "WER", "better": "lower", "fmt": "{:.2f}"}, "rtfx": {"label": "RTFx", "better": "higher", "fmt": "{:.2f}"}, } VIEW_MODE_COLUMNS = { "Model families": "model_family", "Individual models": "model", } DEFAULT_VIEW_MODE = "Model families" # ============================================================================= # Language Normalization # ============================================================================= LANGUAGE_NAME_MAPPING = _load_json("language_name_mapping.json") # ============================================================================= # Geographic Mappings # ============================================================================= # Language to country mapping for Africa map (using ISO 3166-1 alpha-3 codes) LANGUAGE_COUNTRY_MAP = _load_json("language_country_map.json") # Country code to name mapping COUNTRY_NAMES = _load_json("country_names.json") # All African countries (ISO 3166-1 alpha-3 -> name), used to show "No coverage" # on the map for countries without benchmark data. AFRICAN_COUNTRIES = _load_json("african_countries.json") # Language to countries mapping (full country names) # Used for language metadata and region lookups LANGUAGE_TO_COUNTRIES_MAP: dict[str, list[str]] = _load_json("language_to_countries_map.json") # Country to African region mapping (geographical) COUNTRY_TO_REGION_MAP: dict[str, str] = _load_json("country_to_region_map.json") # ============================================================================= # Visualization Interpretation Text # ============================================================================= INTERPRETATIONS = { 'speed_accuracy': """ - Each bubble represents a specific model; **bubble size = model parameter count** - **X-axis (WER)**: Left is better (more accurate) - **Y-axis (RTFx)**: Up is better (faster processing) - **Top-left quadrant (⭐)**: Ideal zone - fast AND accurate models - Gray dashed lines show median values for reference - Uses median values to reduce impact of outliers - Hover over bubbles to see exact parameter counts (e.g., 1.5B, 300M) """, 'leaderboard': """ - **No language selected**: Shows model families (aggregated across all languages) - **Language(s) selected**: Shows top 15 individual models for those languages - The horizontal bars show the median Word Error Rate (WER) - Lower WER values (left side) indicate better accuracy - Error bars represent the standard deviation, showing variability - Bar colors correspond to each model family's assigned color - Hover over bars to see additional metrics like RTFx (speed) and total samples evaluated - Uses median instead of mean to reduce impact of outliers """, 'cer_leaderboard': """ - **No language selected**: Shows model families (aggregated across all languages) - **Language(s) selected**: Shows top 15 individual models for those languages - The horizontal bars show the median Character Error Rate (CER) - Lower CER values (left side) indicate better accuracy - CER is especially important for agglutinative and low-resource languages - Error bars represent the standard deviation, showing variability - Bar colors correspond to each model family's assigned color - Hover over bars to see additional metrics like WER, RTFx (speed) and total samples evaluated - Uses median instead of mean to reduce impact of outliers """, 'correlation': """ - Each point represents one evaluation result - Strong positive correlation means CER and WER move together - Models with high character errors typically also have high word errors - The trend line shows the overall relationship - **Below the line**: Models make more accurate character-level predictions (phonetically closer errors) - **Above the line**: Models make more severe character-level errors per word mistake - Can be filtered by language to analyze specific language patterns """, 'consistency': """ - Coefficient of Variation (CV) = (Standard Deviation / Median) × 100% - **Lower CV** = more consistent performance across different languages - **Higher CV** = performance varies widely depending on the language - Bar colors correspond to each model family's assigned color - Important for production deployment - you want consistent models - Outliers have been removed using IQR method for more robust analysis - Uses median instead of mean for more robust central tendency measure """ }