update: interface

Makefile

@@ -1,18 +0,0 @@
-.PHONY: style format
-
-
-style:
-	python -m black --line-length 119 .
-	python -m isort .
-	ruff check --fix .
-
-
-quality:
-	python -m black --check --line-length 119 .
-	python -m isort --check-only .
-	ruff check .
-
-
-.PHONY: eval
-eval:
-	python -m src.phoneme_eval

README.md

@@ -1,48 +1,56 @@
----
-title: Phoneme Detection Leaderboard
-emoji: 🥇
-colorFrom: green
-colorTo: indigo
-sdk: gradio
-app_file: app.py
-pinned: true
-license: apache-2.0
-short_description: Duplicate this leaderboard to initialize your own!
-sdk_version: 5.43.1
-tags:
-- leaderboard
----
-
-# Start the configuration
-
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
+# Phoneme Detection Leaderboard
+
+A clean, simplified phoneme detection leaderboard based on the open_asr_leaderboard interface.
+
+## Features
+
+- **Clean Interface**: Uses the same interface structure as open_asr_leaderboard
+- **Phoneme Evaluation**: Evaluates models on phoneme recognition tasks
+- **Multiple Datasets**: Supports evaluation on multiple phoneme datasets
+- **Model Request System**: Allows users to request evaluation of new models
+
+## Structure
+
+```
+├── app.py                 # Main Gradio application
+├── constants.py          # Constants and text definitions
+├── utils_display.py      # Display utilities and column definitions
+├── init.py              # Initialization and hub integration
+├── phoneme_eval.py      # Core phoneme evaluation logic
+├── utils/               # Utility modules
+│   ├── load_model.py    # Model loading and inference
+│   ├── audio_process.py # Audio processing and PER calculation
+│   └── cmu_process.py   # CMU to IPA conversion
+├── requirements.txt     # Python dependencies
+└── README.md           # This file
 ```
 
-Request files are created automatically by this tool.
+## Usage
+
+1. Install dependencies:
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+2. Run the application:
+   ```bash
+   python app.py
+   ```
+
+3. Run evaluation:
+   ```bash
+   python phoneme_eval.py
+   ```
+
+## Evaluation
+
+The leaderboard evaluates models on:
+- **PER (Phoneme Error Rate)**: Lower is better
+- **Average Duration**: Processing time per sample
 
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+Models are ranked by Average PER across all datasets.
 
-# Code logic for more complex edits
+## Datasets
 
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+- `phoneme_asr`: General phoneme recognition dataset
+- `kids_phoneme_md`: Children's speech phoneme dataset

app.py

@@ -1,15 +1,32 @@
+import gradio as gr
+import pandas as pd
+import json
 import os
 import glob
-import json
-import pandas as pd
-import gradio as gr
+from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS, LEADERBOARD_CSS
+from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
+from utils_display import PhonemeEvalColumn, fields, make_clickable_model, styled_error, styled_message
+import numpy as np
+from datetime import datetime, timezone
+
+LAST_UPDATED = "Oct 2nd 2025"
 
+# Global variable to store detailed benchmark data
+benchmark_details = {}
 
-ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
-EVAL_RESULTS_DIR = os.path.join(ROOT_DIR, "eval-results")
+# Directory for evaluation results
+EVAL_RESULTS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "eval-results")
 
+column_names = {
+    "model": "Model",
+    "avg_per": "Average PER ⬇️",    
+    "avg_duration": "Avg Duration (s)",
+    "per_phoneme_asr": "PER phoneme_asr",
+    "per_kids_phoneme_md": "PER kids_phoneme_md",
+}
 
 def load_results(results_dir: str) -> pd.DataFrame:
+    """Load results from JSON files in the results directory"""
     rows = []
     all_dataset_keys = set()
     
@@ -62,10 +79,9 @@ def load_results(results_dir: str) -> pd.DataFrame:
             avg_dur = sum(dur_values) / len(dur_values) if dur_values else None
 
             row = {
-                "Model": model_name,
-                "Avg PER": avg_per,
+                "Model": make_clickable_model(model_name),
+                "Average PER ⬇️": avg_per,
                 "Avg Duration (s)": avg_dur,
-                "_file": os.path.basename(path),
             }
             row.update(per_values)
             rows.append(row)
@@ -76,56 +92,198 @@ def load_results(results_dir: str) -> pd.DataFrame:
     df = pd.DataFrame(rows)
     if df.empty:
         # Create default columns based on discovered datasets
-        default_cols = ["Model", "Avg PER", "Avg Duration (s)"]
+        default_cols = ["Model", "Average PER ⬇️", "Avg Duration (s)"]
         for key in sorted(all_dataset_keys):
             display_name = dataset_display_names[key]
             default_cols.insert(-2, f"PER {display_name}")
         return pd.DataFrame(columns=default_cols)
     
-    df = df.sort_values(by=["Avg PER"], ascending=True, na_position="last")
+    df = df.sort_values(by=["Average PER ⬇️"], ascending=True, na_position="last")
     return df.reset_index(drop=True)
 
+# Load initial data
+try:
+    eval_queue_repo, requested_models, csv_results = load_all_info_from_dataset_hub()
+    if csv_results and csv_results.exists():
+        original_df = pd.read_csv(csv_results)
+        # Format the columns
+        def formatter(x):
+            if type(x) is str:
+                x = x
+            elif x == -1:
+                x = "NA"
+            else: 
+                x = round(x, 2)
+            return x
 
-def build_interface():
-    with gr.Blocks() as demo:
-        gr.Markdown("# Simple Phoneme Leaderboard")
-        info = gr.Markdown(f"Results directory: `{EVAL_RESULTS_DIR}`")
-        
-        # Get initial data to determine columns dynamically
-        initial_df = load_results(EVAL_RESULTS_DIR)
-        if not initial_df.empty:
-            headers = list(initial_df.columns)
-            # Remove internal columns
-            headers = [h for h in headers if not h.startswith('_')]
-        else:
-            headers = ["Model", "Avg PER", "Avg Duration (s)"]
+        for col in original_df.columns:
+            if col == "model":
+                original_df[col] = original_df[col].apply(lambda x: make_clickable_model(x))
+            else:
+                original_df[col] = original_df[col].apply(formatter)
+        # Only rename columns that exist in the dataframe
+        existing_columns = {k: v for k, v in column_names.items() if k in original_df.columns}
+        original_df.rename(columns=existing_columns, inplace=True)
+        if 'Average PER ⬇️' in original_df.columns:
+            original_df.sort_values(by='Average PER ⬇️', inplace=True)
+    else:
+        # Fallback to local results
+        original_df = load_results(EVAL_RESULTS_DIR)
+except Exception as e:
+    print(f"Error loading data: {e}")
+    # Fallback to local results
+    original_df = load_results(EVAL_RESULTS_DIR)
+
+# If no data is loaded, create a sample empty dataframe with proper columns
+if original_df.empty:
+    print("No results found. Creating empty dataframe with sample data...")
+    # Create sample data to demonstrate the interface
+    sample_data = {
+        "Model": [make_clickable_model("sample/hubert-base"), make_clickable_model("sample/whisper-base")],
+        "Average PER ⬇️": [15.2, 18.5],
+        "Avg Duration (s)": [0.12, 0.15],
+        "PER phoneme_asr": [14.8, 17.2],
+        "PER kids_phoneme_md": [15.6, 19.8]
+    }
+    original_df = pd.DataFrame(sample_data)
+    print("Sample data created for demonstration.")
+
+COLS = [c.name for c in fields(PhonemeEvalColumn)]
+TYPES = [c.type for c in fields(PhonemeEvalColumn)]
+
+def request_model(model_text, chb_phoneme_asr, chb_kids_phoneme_md):
+    
+    # Determine the selected checkboxes
+    dataset_selection = []
+    if chb_phoneme_asr:
+        dataset_selection.append("phoneme_asr")
+    if chb_kids_phoneme_md:
+        dataset_selection.append("kids_phoneme_md")
+
+    if len(dataset_selection) == 0:
+        return styled_error("You need to select at least one dataset")
         
-        table = gr.Dataframe(headers=headers, row_count=5)
+    base_model_on_hub, error_msg = is_model_on_hub(model_text)
 
-        def refresh():
-            df = load_results(EVAL_RESULTS_DIR)
-            if df.empty:
-                return df
-            
-            # Get the column order from the dataframe
-            cols = [c for c in df.columns if not c.startswith('_')]
+    if not base_model_on_hub:
+        return styled_error(f"Base model '{model_text}' {error_msg}")
+    
+    # Construct the output dictionary
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    required_datasets = ', '.join(dataset_selection)
+    eval_entry = {
+        "date": current_time,
+        "model": model_text,
+        "datasets_selected": required_datasets
+    }
+    
+    # Prepare file path 
+    DIR_OUTPUT_REQUESTS.mkdir(parents=True, exist_ok=True)
+    
+    fn_datasets = '@ '.join(dataset_selection)
+    filename = model_text.replace("/","@") + "@@" + fn_datasets 
+    if filename in requested_models:
+        return styled_error(f"A request for this model '{model_text}' and dataset(s) was already made.")
+    try:
+        filename_ext = filename + ".txt"
+        out_filepath = DIR_OUTPUT_REQUESTS / filename_ext
+
+        # Write the results to a text file
+        with open(out_filepath, "w") as f:
+            f.write(json.dumps(eval_entry))
             
-            # Ensure all columns exist for the dataframe component
-            for c in cols:
-                if c not in df.columns:
-                    df[c] = None
-            return df[cols].round(3)
+        upload_file(filename, out_filepath)
+        
+        # Include file in the list of uploaded files
+        requested_models.append(filename)
+        
+        # Remove the local file
+        out_filepath.unlink()
+
+        return styled_message("🤗 Your request has been submitted and will be evaluated soon!</p>")
+    except Exception as e:
+        return styled_error(f"Error submitting request!")
 
-        btn = gr.Button("Refresh")
-        btn.click(fn=refresh, outputs=table)
+def filter_main_table(show_proprietary=True):
+    filtered_df = original_df.copy()
+    
+    # Filter proprietary models if needed
+    if not show_proprietary and "License" in filtered_df.columns:
+        # Keep only models with "Open" license
+        filtered_df = filtered_df[filtered_df["License"] == "Open"]
+        
+    return filtered_df
 
-        # Auto-load on start
-        table.value = refresh()
-    return demo
+def refresh_results():
+    """Refresh the results from the eval-results directory"""
+    updated_df = load_results(EVAL_RESULTS_DIR)
+    return updated_df
 
+with gr.Blocks(css=LEADERBOARD_CSS) as demo:
+    # gr.HTML(BANNER, elem_id="banner")
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
-if __name__ == "__main__":
-    demo = build_interface()
-    demo.queue().launch()
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 Leaderboard", elem_id="phoneme-benchmark-tab-table", id=0):
+            leaderboard_table = gr.components.Dataframe(
+                value=original_df,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+            with gr.Row():
+                show_proprietary_checkbox = gr.Checkbox(
+                    label="Show proprietary models",
+                    value=True,
+                    elem_id="show-proprietary-checkbox"
+                )
+                refresh_button = gr.Button("🔄 Refresh Results", variant="secondary")
+            
+            # Connect checkbox to the filtering function
+            show_proprietary_checkbox.change(
+                filter_main_table,
+                inputs=[show_proprietary_checkbox],
+                outputs=leaderboard_table
+            )
+            
+            # Connect refresh button
+            refresh_button.click(
+                refresh_results,
+                outputs=leaderboard_table
+            )
 
+        with gr.TabItem("📈 Metrics", elem_id="phoneme-benchmark-tab-table", id=1):
+            gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("✉️✨ Request a model here!", elem_id="phoneme-benchmark-tab-table", id=2):
+            with gr.Column():
+                gr.Markdown("# ✉️✨ Request results for a new model here!", elem_classes="markdown-text")
+            with gr.Column():
+                gr.Markdown("Select datasets:", elem_classes="markdown-text")
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name (user_name/model_name)")
+                    chb_phoneme_asr = gr.Checkbox(label="phoneme_asr dataset", value=True)
+                    chb_kids_phoneme_md = gr.Checkbox(label="kids_phoneme_md dataset", value=True)
+                with gr.Column():
+                    mdw_submission_result = gr.Markdown()
+                    btn_submitt = gr.Button(value="🚀 Request")
+                    btn_submitt.click(request_model, 
+                                      [model_name_textbox, chb_phoneme_asr, chb_kids_phoneme_md], 
+                                      mdw_submission_result)
+        # add an about section
+        with gr.TabItem("🤗 About", elem_id="phoneme-benchmark-tab-table", id=3):
+            gr.Markdown("## About", elem_classes="markdown-text")
+
+    gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text")
+    
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            gr.Textbox(
+                value=CITATION_TEXT, lines=7,
+                label="Copy the BibTeX snippet to cite this source",
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
 
+demo.launch(ssr_mode=False)

app_default.py

@@ -1,463 +0,0 @@
-import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-import os
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    COLS,
-    AutoEvalColumn,
-    fields,
-)
-from src.about import Tasks
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-# Import simple leaderboard functionality
-import glob
-import json
-from functools import lru_cache
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation (prefer local JSONs, fall back to Hub)
-def _has_local_json(path: str) -> bool:
-    try:
-        return os.path.isdir(path) and any(str(f).endswith(".json") for f in os.listdir(path))
-    except Exception:
-        return False
-
-if not _has_local_json(EVAL_REQUESTS_PATH):
-    try:
-        print(EVAL_REQUESTS_PATH)
-        snapshot_download(
-            repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-        )
-    except Exception:
-        pass
-
-if not _has_local_json(EVAL_RESULTS_PATH):
-    try:
-        print(EVAL_RESULTS_PATH)
-        snapshot_download(
-            repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-        )
-    except Exception:
-        pass
-
-
-# Build benchmark and evaluation queue column metadata
-BENCHMARK_COLS = [f"{task.value.col_name} ({task.name})" for task in Tasks]
-
-EVAL_COLS = [
-    "Model",
-    "Model sha",
-    "status",
-    "precision",
-    "weight_type",
-    "model_type",
-    "likes",
-    "params",
-    "license",
-    "submitted_time",
-]
-
-EVAL_TYPES = [
-    "markdown",  # Model
-    "str",       # Model sha
-    "str",       # status
-    "str",       # precision
-    "str",       # weight_type
-    "str",       # model_type
-    "number",    # likes
-    "number",    # params
-    "str",       # license
-    "str",       # submitted_time
-]
-
-# Hide all models from the leaderboard view
-LEADERBOARD_DF = pd.DataFrame(columns=COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-@lru_cache(maxsize=1)
-def _get_simple_dataset_keys(results_dir: str) -> tuple:
-    """Cache dataset keys to avoid repeated file scanning."""
-    all_dataset_keys = set()
-    if not os.path.isdir(results_dir):
-        return tuple()
-    
-    for path in glob.glob(os.path.join(results_dir, "*.json")):
-        try:
-            with open(path, "r", encoding="utf-8") as f:
-                data = json.load(f)
-            res = data.get("results", {})
-            all_dataset_keys.update(res.keys())
-        except Exception:
-            continue
-    
-    return tuple(sorted(all_dataset_keys))
-
-def load_simple_results(results_dir: str) -> pd.DataFrame:
-    """Load and process evaluation results from JSON files for simple leaderboard with caching."""
-    rows = []
-    all_dataset_keys = set(_get_simple_dataset_keys(results_dir))
-    
-    if not all_dataset_keys:
-        return pd.DataFrame(columns=["Model", "Avg PER", "Avg Duration (s)"])
-
-    # Use dataset keys directly as display names
-    dataset_display_names = {key: key for key in all_dataset_keys}
-
-    # Single pass: extract data with optimized processing
-    for path in glob.glob(os.path.join(results_dir, "*.json")):
-        try:
-            with open(path, "r", encoding="utf-8") as f:
-                data = json.load(f)
-            cfg = data.get("config", {})
-            res = data.get("results", {})
-
-            model_name = cfg.get("model_name", "unknown")
-            
-            # Extract PER for each dataset dynamically
-            per_values = {}
-            dur_values = []
-            
-            for dataset_key in all_dataset_keys:
-                dataset_data = res.get(dataset_key, {})
-                per_value = dataset_data.get("per") if dataset_data else None
-                dur_value = dataset_data.get("avg_duration") if dataset_data else None
-                
-                display_name = dataset_display_names[dataset_key]
-                per_values[f"PER {display_name}"] = per_value
-                
-                if dur_value is not None:
-                    dur_values.append(dur_value)
-            
-            # Calculate average PER across all datasets
-            per_vals = [v for v in per_values.values() if v is not None]
-            avg_per = sum(per_vals) / len(per_vals) if per_vals else None
-            
-            # Calculate average duration
-            avg_dur = sum(dur_values) / len(dur_values) if dur_values else None
-
-            row = {
-                "Model": model_name,
-                "Avg PER": avg_per,
-                "Avg Duration (s)": avg_dur,
-                "_file": os.path.basename(path),
-            }
-            row.update(per_values)
-            rows.append(row)
-            
-        except Exception:
-            continue
-
-    df = pd.DataFrame(rows)
-    if df.empty:
-        # Create default columns based on discovered datasets
-        default_cols = ["Model", "Avg PER", "Avg Duration (s)"]
-        for key in sorted(all_dataset_keys):
-            display_name = dataset_display_names[key]
-            default_cols.insert(-2, f"PER {display_name}")
-        return pd.DataFrame(columns=default_cols)
-    
-    df = df.sort_values(by=["Avg PER"], ascending=True, na_position="last")
-    return df.reset_index(drop=True)
-
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        dataframe = pd.DataFrame(columns=[c.name for c in fields(AutoEvalColumn)])
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 Phoneme Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📊 Simple Results", elem_id="simple-results-tab", id=1):
-            gr.Markdown("## 🎯 Phoneme Detection Results")
-            gr.Markdown("Compare phoneme recognition models across different datasets")
-            
-            # Stats section for simple results
-            with gr.Row():
-                simple_total_models = gr.HTML(
-                    '<div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 1rem; border-radius: 10px; text-align: center; min-width: 150px;"><div style="font-size: 1.5rem; font-weight: bold;">-</div><div style="font-size: 0.9rem; opacity: 0.9;">Total Models</div></div>'
-                )
-                simple_best_per = gr.HTML(
-                    '<div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 1rem; border-radius: 10px; text-align: center; min-width: 150px;"><div style="font-size: 1.5rem; font-weight: bold;">-</div><div style="font-size: 0.9rem; opacity: 0.9;">Best PER</div></div>'
-                )
-                simple_avg_duration = gr.HTML(
-                    '<div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 1rem; border-radius: 10px; text-align: center; min-width: 150px;"><div style="font-size: 1.5rem; font-weight: bold;">-</div><div style="font-size: 0.9rem; opacity: 0.9;">Avg Duration</div></div>'
-                )
-            
-            # Get initial data to determine columns dynamically
-            initial_df = load_simple_results(EVAL_RESULTS_PATH)
-            if not initial_df.empty:
-                headers = list(initial_df.columns)
-                # Remove internal columns
-                headers = [h for h in headers if not h.startswith('_')]
-            else:
-                headers = ["Model", "Avg PER", "Avg Duration (s)"]
-            
-            with gr.Row():
-                with gr.Column(scale=4):
-                    simple_table = gr.Dataframe(
-                        headers=headers, 
-                        row_count=10,
-                        label="🏆 Model Performance Leaderboard",
-                        interactive=False
-                    )
-                
-                with gr.Column(scale=1):
-                    refresh_btn = gr.Button("🔄 Refresh Data", variant="primary")
-                    
-                    # Export options
-                    with gr.Accordion("📥 Export Data", open=False):
-                        export_csv = gr.Button("📄 Export CSV", variant="secondary")
-                        export_json = gr.Button("📋 Export JSON", variant="secondary")
-
-            def refresh_simple():
-                """Refresh the simple leaderboard data with enhanced stats."""
-                df = load_simple_results(EVAL_RESULTS_PATH)
-                
-                if df.empty:
-                    return df, "No data", "No data", "No data"
-                
-                # Get the column order from the dataframe
-                cols = [c for c in df.columns if not c.startswith('_')]
-                
-                # Ensure all columns exist for the dataframe component
-                for c in cols:
-                    if c not in df.columns:
-                        df[c] = None
-                
-                # Calculate enhanced stats
-                total_models = len(df)
-                best_per_val = df['Avg PER'].min() if 'Avg PER' in df.columns and not df['Avg PER'].isna().all() else "N/A"
-                avg_duration_val = df['Avg Duration (s)'].mean() if 'Avg Duration (s)' in df.columns and not df['Avg Duration (s)'].isna().all() else "N/A"
-                
-                # Format stats
-                best_per_str = f"{best_per_val:.2f}" if isinstance(best_per_val, (int, float)) else str(best_per_val)
-                avg_duration_str = f"{avg_duration_val:.2f}s" if isinstance(avg_duration_val, (int, float)) else str(avg_duration_val)
-                
-                return (
-                    df[cols].round(3),
-                    f'<div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 1rem; border-radius: 10px; text-align: center; min-width: 150px;"><div style="font-size: 1.5rem; font-weight: bold;">{total_models}</div><div style="font-size: 0.9rem; opacity: 0.9;">Total Models</div></div>',
-                    f'<div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 1rem; border-radius: 10px; text-align: center; min-width: 150px;"><div style="font-size: 1.5rem; font-weight: bold;">{best_per_str}</div><div style="font-size: 0.9rem; opacity: 0.9;">Best PER</div></div>',
-                    f'<div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 1rem; border-radius: 10px; text-align: center; min-width: 150px;"><div style="font-size: 1.5rem; font-weight: bold;">{avg_duration_str}</div><div style="font-size: 0.9rem; opacity: 0.9;">Avg Duration</div></div>'
-                )
-
-            def export_simple_csv():
-                """Export simple results as CSV."""
-                df = load_simple_results(EVAL_RESULTS_PATH)
-                if df.empty:
-                    return None
-                cols = [c for c in df.columns if not c.startswith('_')]
-                return df[cols].round(3)
-
-            def export_simple_json():
-                """Export simple results as JSON."""
-                df = load_simple_results(EVAL_RESULTS_PATH)
-                if df.empty:
-                    return None
-                cols = [c for c in df.columns if not c.startswith('_')]
-                return df[cols].round(3).to_json(orient='records', indent=2)
-
-            # Connect events
-            refresh_btn.click(
-                fn=refresh_simple, 
-                outputs=[simple_table, simple_total_models, simple_best_per, simple_avg_duration]
-            )
-            
-            export_csv.click(
-                fn=export_simple_csv,
-                outputs=gr.File(label="Download CSV")
-            )
-            
-            export_json.click(
-                fn=export_simple_json,
-                outputs=gr.File(label="Download JSON")
-            )
-
-            # Auto-load on start
-            simple_table.value, simple_total_models.value, simple_best_per.value, simple_avg_duration.value = refresh_simple()
-            
-            # Enhanced help section
-            with gr.Accordion("ℹ️ About this Leaderboard", open=False):
-                gr.Markdown("""
-                ## 📊 Understanding the Results
-                
-                **Performance Metrics:**
-                - **PER (Phoneme Error Rate)**: Lower values indicate better performance
-                - **Avg Duration**: Processing time per sample (lower is faster)
-                - **Models are ranked by average PER across all datasets**
-                
-                **Datasets Evaluated:**
-                - `phoneme_asr`: General phoneme recognition dataset
-                - `kids_phoneme_md`: Kids' phoneme recognition dataset
-                
-                **How to Interpret:**
-                - **PER**: Percentage of phonemes incorrectly recognized (0% = perfect)
-                - **Duration**: Time efficiency (important for real-time applications)
-                - **Average PER**: Overall model performance across all datasets
-                
-                **Tips for Model Selection:**
-                - Choose models with low PER for accuracy-critical applications
-                - Consider duration for real-time or resource-constrained environments
-                - Balance between accuracy (PER) and speed (Duration) based on your needs
-                """)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=["Pretrained", "Fine-tuned", "Merge", "Other"],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=["float16", "bfloat16", "float32", "int8", "int4"],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=["Original", "Delta", "Adapter"],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()

app_simple.py

@@ -1,318 +0,0 @@
-import os
-import glob
-import json
-import pandas as pd
-import gradio as gr
-from typing import Optional, Dict, List
-import time
-from functools import lru_cache
-
-ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
-EVAL_RESULTS_DIR = os.path.join(ROOT_DIR, "eval-results")
-
-
-@lru_cache(maxsize=1)
-def _get_dataset_keys(results_dir: str) -> tuple:
-    """Cache dataset keys to avoid repeated file scanning."""
-    all_dataset_keys = set()
-    if not os.path.isdir(results_dir):
-        return tuple()
-    
-    for path in glob.glob(os.path.join(results_dir, "*.json")):
-        try:
-            with open(path, "r", encoding="utf-8") as f:
-                data = json.load(f)
-            res = data.get("results", {})
-            all_dataset_keys.update(res.keys())
-        except Exception:
-            continue
-    
-    return tuple(sorted(all_dataset_keys))
-
-def load_results(results_dir: str) -> pd.DataFrame:
-    """
-    Load and process evaluation results from JSON files.
-    Dynamically handles any number of datasets with caching for performance.
-    """
-    rows = []
-    all_dataset_keys = set(_get_dataset_keys(results_dir))
-    
-    if not all_dataset_keys:
-        return pd.DataFrame(columns=["Model", "Avg PER", "Avg Duration (s)"])
-
-    # Use dataset keys directly as display names
-    dataset_display_names = {key: key for key in all_dataset_keys}
-
-    # Single pass: extract data with optimized processing
-    for path in glob.glob(os.path.join(results_dir, "*.json")):
-        try:
-            with open(path, "r", encoding="utf-8") as f:
-                data = json.load(f)
-            cfg = data.get("config", {})
-            res = data.get("results", {})
-
-            model_name = cfg.get("model_name", "unknown")
-            
-            # Extract PER for each dataset dynamically
-            per_values = {}
-            dur_values = []
-            
-            for dataset_key in all_dataset_keys:
-                dataset_data = res.get(dataset_key, {})
-                per_value = dataset_data.get("per") if dataset_data else None
-                dur_value = dataset_data.get("avg_duration") if dataset_data else None
-                
-                display_name = dataset_display_names[dataset_key]
-                per_values[f"PER {display_name}"] = per_value
-                
-                if dur_value is not None:
-                    dur_values.append(dur_value)
-            
-            # Calculate average PER across all datasets
-            per_vals = [v for v in per_values.values() if v is not None]
-            avg_per = sum(per_vals) / len(per_vals) if per_vals else None
-            
-            # Calculate average duration
-            avg_dur = sum(dur_values) / len(dur_values) if dur_values else None
-
-            row = {
-                "Model": model_name,
-                "Avg PER": avg_per,
-                "Avg Duration (s)": avg_dur,
-                "_file": os.path.basename(path),
-            }
-            row.update(per_values)
-            rows.append(row)
-            
-        except Exception:
-            continue
-
-    df = pd.DataFrame(rows)
-    if df.empty:
-        # Create default columns based on discovered datasets
-        default_cols = ["Model", "Avg PER", "Avg Duration (s)"]
-        for key in sorted(all_dataset_keys):
-            display_name = dataset_display_names[key]
-            default_cols.insert(-2, f"PER {display_name}")
-        return pd.DataFrame(columns=default_cols)
-    
-    df = df.sort_values(by=["Avg PER"], ascending=True, na_position="last")
-    return df.reset_index(drop=True)
-
-
-def build_interface():
-    """Build the optimized Gradio interface for the phoneme leaderboard."""
-    
-    # Custom CSS for better styling
-    custom_css = """
-    .gradio-container {
-        max-width: 1200px !important;
-        margin: 0 auto !important;
-    }
-    .leaderboard-header {
-        text-align: center;
-        margin-bottom: 2rem;
-    }
-    .stats-container {
-        display: flex;
-        gap: 1rem;
-        margin-bottom: 1rem;
-        flex-wrap: wrap;
-    }
-    .stat-card {
-        background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
-        color: white;
-        padding: 1rem;
-        border-radius: 10px;
-        text-align: center;
-        min-width: 150px;
-        flex: 1;
-    }
-    .stat-value {
-        font-size: 1.5rem;
-        font-weight: bold;
-        margin-bottom: 0.5rem;
-    }
-    .stat-label {
-        font-size: 0.9rem;
-        opacity: 0.9;
-    }
-    .table-container {
-        margin-top: 1rem;
-    }
-    .refresh-btn {
-        background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
-        color: white;
-        border: none;
-        padding: 0.5rem 1rem;
-        border-radius: 5px;
-        cursor: pointer;
-    }
-    """
-    
-    with gr.Blocks(
-        title="Phoneme Detection Leaderboard",
-        css=custom_css,
-        theme=gr.themes.Soft()
-    ) as demo:
-        
-        # Header section
-        with gr.Column(elem_classes="leaderboard-header"):
-            gr.Markdown("# 🎯 Phoneme Detection Leaderboard")
-            gr.Markdown("Compare phoneme recognition models across different datasets")
-        
-        # Stats section
-        with gr.Row(elem_classes="stats-container"):
-            total_models = gr.HTML(
-                '<div class="stat-card"><div class="stat-value" id="total-models">-</div><div class="stat-label">Total Models</div></div>',
-                elem_id="total-models-card"
-            )
-            best_per = gr.HTML(
-                '<div class="stat-card"><div class="stat-value" id="best-per">-</div><div class="stat-label">Best PER</div></div>',
-                elem_id="best-per-card"
-            )
-            avg_duration = gr.HTML(
-                '<div class="stat-card"><div class="stat-value" id="avg-duration">-</div><div class="stat-label">Avg Duration</div></div>',
-                elem_id="avg-duration-card"
-            )
-        
-        # Main content
-        with gr.Row():
-            with gr.Column(scale=4):
-                # Get initial data to determine columns dynamically
-                initial_df = load_results(EVAL_RESULTS_DIR)
-                if not initial_df.empty:
-                    headers = list(initial_df.columns)
-                    # Remove internal columns
-                    headers = [h for h in headers if not h.startswith('_')]
-                else:
-                    headers = ["Model", "Avg PER", "Avg Duration (s)"]
-                
-                table = gr.Dataframe(
-                    headers=headers, 
-                    row_count=10,
-                    label="🏆 Model Performance Leaderboard",
-                    interactive=False,
-                    elem_classes="table-container"
-                )
-            
-            with gr.Column(scale=1):
-                refresh_btn = gr.Button(
-                    "🔄 Refresh Data", 
-                    variant="primary",
-                    elem_classes="refresh-btn"
-                )
-                
-                # Quick stats
-                with gr.Accordion("📊 Quick Stats", open=True):
-                    stats_display = gr.HTML("Loading statistics...")
-                
-                # Export options
-                with gr.Accordion("📥 Export Data", open=False):
-                    export_csv = gr.Button("📄 Export as CSV", variant="secondary")
-                    export_json = gr.Button("📋 Export as JSON", variant="secondary")
-
-        def refresh():
-            """Refresh the leaderboard data with performance optimization."""
-            start_time = time.time()
-            df = load_results(EVAL_RESULTS_DIR)
-            
-            if df.empty:
-                return df, "No data available", "No data available", "No data available"
-            
-            # Get the column order from the dataframe
-            cols = [c for c in df.columns if not c.startswith('_')]
-            
-            # Ensure all columns exist for the dataframe component
-            for c in cols:
-                if c not in df.columns:
-                    df[c] = None
-            
-            # Calculate stats
-            total_models = len(df)
-            best_per_val = df['Avg PER'].min() if 'Avg PER' in df.columns and not df['Avg PER'].isna().all() else "N/A"
-            avg_duration_val = df['Avg Duration (s)'].mean() if 'Avg Duration (s)' in df.columns and not df['Avg Duration (s)'].isna().all() else "N/A"
-            
-            # Format stats
-            best_per_str = f"{best_per_val:.2f}" if isinstance(best_per_val, (int, float)) else str(best_per_val)
-            avg_duration_str = f"{avg_duration_val:.2f}s" if isinstance(avg_duration_val, (int, float)) else str(avg_duration_val)
-            
-            load_time = time.time() - start_time
-            
-            return (
-                df[cols].round(3),
-                f"<div class='stat-card'><div class='stat-value'>{total_models}</div><div class='stat-label'>Total Models</div></div>",
-                f"<div class='stat-card'><div class='stat-value'>{best_per_str}</div><div class='stat-label'>Best PER</div></div>",
-                f"<div class='stat-card'><div class='stat-value'>{avg_duration_str}</div><div class='stat-label'>Avg Duration</div></div>"
-            )
-
-        def export_csv_data():
-            """Export data as CSV."""
-            df = load_results(EVAL_RESULTS_DIR)
-            if df.empty:
-                return None
-            cols = [c for c in df.columns if not c.startswith('_')]
-            return df[cols].round(3)
-
-        def export_json_data():
-            """Export data as JSON."""
-            df = load_results(EVAL_RESULTS_DIR)
-            if df.empty:
-                return None
-            cols = [c for c in df.columns if not c.startswith('_')]
-            return df[cols].round(3).to_json(orient='records', indent=2)
-
-        # Connect events
-        refresh_btn.click(
-            fn=refresh, 
-            outputs=[table, total_models, best_per, avg_duration]
-        )
-        
-        export_csv.click(
-            fn=export_csv_data,
-            outputs=gr.File(label="Download CSV")
-        )
-        
-        export_json.click(
-            fn=export_json_data,
-            outputs=gr.File(label="Download JSON")
-        )
-
-        # Auto-load on start
-        table.value, total_models.value, best_per.value, avg_duration.value = refresh()
-        
-        # Help section
-        with gr.Accordion("ℹ️ About this Leaderboard", open=False):
-            gr.Markdown("""
-            ## 📊 Understanding the Results
-            
-            **Performance Metrics:**
-            - **PER (Phoneme Error Rate)**: Lower values indicate better performance
-            - **Avg Duration**: Processing time per sample (lower is faster)
-            - **Models are ranked by average PER across all datasets**
-            
-            **Datasets Evaluated:**
-            - `phoneme_asr`: General phoneme recognition dataset
-            - `kids_phoneme_md`: Kids' phoneme recognition dataset
-            
-            **How to Interpret:**
-            - **PER**: Percentage of phonemes incorrectly recognized (0% = perfect)
-            - **Duration**: Time efficiency (important for real-time applications)
-            - **Average PER**: Overall model performance across all datasets
-            
-            **Tips for Model Selection:**
-            - Choose models with low PER for accuracy-critical applications
-            - Consider duration for real-time or resource-constrained environments
-            - Balance between accuracy (PER) and speed (Duration) based on your needs
-            """)
-
-    return demo
-
-
-if __name__ == "__main__":
-    demo = build_interface()
-    demo.queue().launch(
-        server_name="0.0.0.0",
-        server_port=7860,
-        share=False
-    )

constants.py

@@ -0,0 +1,87 @@
+from pathlib import Path
+
+# Directory where request by models are stored
+DIR_OUTPUT_REQUESTS = Path("requested_models")
+EVAL_REQUESTS_PATH = Path("eval_requests")
+
+##########################
+# Text definitions       #
+##########################
+
+banner_url = "https://huggingface.co/datasets/reach-vb/random-images/resolve/main/phoneme_leaderboard.png"
+BANNER = f'<div style="display: flex; justify-content: space-around;"><img src="{banner_url}" alt="Banner" style="width: 40vw; min-width: 300px; max-width: 600px;"> </div>'
+
+TITLE = "<html> <head> <style> h1 {text-align: center;} </style> </head> <body> <h1> 🤗 Phoneme Detection Leaderboard </b> </body> </html>"
+
+INTRODUCTION_TEXT = """📐 The 🤗 Phoneme Detection Leaderboard ranks and evaluates phoneme recognition models 
+    on the Hugging Face Hub. 
+    \nWe report the Average [PER](https://en.wikipedia.org/wiki/Phoneme_error_rate) (⬇️ lower the better) and Average Duration. Models are ranked based on their Average PER, from lowest to highest. Check the 📈 Metrics tab to understand how the models are evaluated. 
+    \nIf you want results for a model that is not listed here, you can submit a request for it to be included ✉️✨. 
+    \nThe leaderboard includes phoneme recognition evaluation across multiple datasets."""
+
+CITATION_TEXT = """@misc{phoneme-detection-leaderboard,
+	title        = {Phoneme Detection Leaderboard},
+	author       = {Your Name and Contributors},
+	year         = 2024,
+	publisher    = {Hugging Face},
+	howpublished = "\\url{https://huggingface.co/spaces/your-org/phoneme-detection-leaderboard}"
+}
+"""
+
+METRICS_TAB_TEXT = """
+Here you will find details about the phoneme recognition metrics and datasets reported in our leaderboard.
+
+## Metrics
+
+Models are evaluated using the Phoneme Error Rate (PER) metric. The PER metric
+is used to assess the accuracy of a phoneme recognition system. Models are ranked in the leaderboard based 
+on their PER, lowest to highest.
+
+### Phoneme Error Rate (PER)
+
+Phoneme Error Rate is used to measure the **accuracy** of automatic phoneme recognition systems. It calculates the percentage 
+of phonemes in the system's output that differ from the reference (correct) phoneme sequence. **A lower PER value indicates higher accuracy**.
+
+The PER is calculated using sequence alignment between predicted and reference phoneme sequences, taking into account:
+- Substitutions (S): predicted phoneme differs from reference
+- Deletions (D): reference phoneme missing in prediction  
+- Insertions (I): predicted phoneme not in reference
+
+```
+PER = (S + D + I) / N * 100
+```
+
+Where N is the total number of reference phonemes.
+
+## How to reproduce our results
+
+The Phoneme Detection Leaderboard is an effort to benchmark open source phoneme recognition models. 
+Along with the Leaderboard we're open-sourcing the codebase used for running these evaluations.
+
+P.S. We'd love to know which other models you'd like us to benchmark next. Contributions are more than welcome! ♥️
+
+## Benchmark datasets
+
+Evaluating Phoneme Recognition systems requires diverse datasets with phonetic transcriptions. We use multiple datasets to obtain robust evaluation scores for each model.
+
+| Dataset | Description | Language | License |
+|---------|-------------|----------|---------|
+| phoneme_asr | General phoneme recognition dataset | English | Open |
+| kids_phoneme_md | Children's speech phoneme dataset | English | Open |
+
+For more details on the individual datasets and how models are evaluated, refer to our documentation.
+"""
+
+LEADERBOARD_CSS = """
+#leaderboard-table th .header-content {
+    white-space: nowrap;
+}
+
+#phoneme-table th .header-content {
+    white-space: nowrap;
+}
+
+#phoneme-table th:hover {
+    background-color: var(--table-row-focus);
+}
+"""

eval-results/{results_1759289565_HuBERT-Base.json → results_1759378937_HuBERT-Base.json}

@@ -1,17 +1,17 @@
 {
   "config": {
-    "model_name": "local/HuBERT-Base",
+    "model_name": "HuBERT-Base",
     "model_dtype": "float32",
     "model_sha": ""
   },
   "results": {
     "phoneme_asr": {
       "per": 79.85359813133437,
-      "avg_duration": 0.5645037651062011
+      "avg_duration": 0.7736877918243408
     },
     "kids_phoneme_md": {
       "per": 71.85295670319688,
-      "avg_duration": 1.0543905973434449
+      "avg_duration": 1.47061448097229
     }
   }
 }

eval-results/{results_1759289565_HuBERT-fine-tuned.json → results_1759378937_HuBERT-fine-tuned.json}

@@ -1,17 +1,17 @@
 {
   "config": {
-    "model_name": "local/HuBERT-fine-tuned",
+    "model_name": "HuBERT-fine-tuned",
     "model_dtype": "float32",
     "model_sha": ""
   },
   "results": {
     "phoneme_asr": {
       "per": 2.774112645808511,
-      "avg_duration": 0.5711040496826172
+      "avg_duration": 0.7994948387145996
     },
     "kids_phoneme_md": {
       "per": 12.210125572986708,
-      "avg_duration": 1.0601478815078735
+      "avg_duration": 1.439890170097351
     }
   }
 }

eval-results/{results_1759289565_Timit.json → results_1759378937_Timit.json}

@@ -1,17 +1,17 @@
 {
   "config": {
-    "model_name": "local/Timit",
+    "model_name": "Timit",
     "model_dtype": "float32",
     "model_sha": ""
   },
   "results": {
     "phoneme_asr": {
       "per": 36.477283094931195,
-      "avg_duration": 0.554583740234375
+      "avg_duration": 0.8033712863922119
     },
     "kids_phoneme_md": {
       "per": 40.59831492610759,
-      "avg_duration": 1.0818484544754028
+      "avg_duration": 1.455029034614563
     }
   }
 }

eval-results/results_1759378937_Whisper.json

@@ -0,0 +1,17 @@
+{
+  "config": {
+    "model_name": "Whisper",
+    "model_dtype": "float32",
+    "model_sha": ""
+  },
+  "results": {
+    "phoneme_asr": {
+      "per": 80.66478307042628,
+      "avg_duration": 1.2233323097229003
+    },
+    "kids_phoneme_md": {
+      "per": 72.25186973830769,
+      "avg_duration": 1.3742226600646972
+    }
+  }
+}

init.py

@@ -0,0 +1,89 @@
+import os 
+from constants import EVAL_REQUESTS_PATH
+from pathlib import Path
+from huggingface_hub import HfApi, Repository
+
+TOKEN_HUB = os.environ.get("TOKEN_HUB", None)
+QUEUE_REPO = os.environ.get("QUEUE_REPO", None)
+QUEUE_PATH = os.environ.get("QUEUE_PATH", None)
+
+hf_api = HfApi(
+    endpoint="https://huggingface.co", 
+    token=TOKEN_HUB, 
+)
+
+def load_all_info_from_dataset_hub():
+    eval_queue_repo = None
+    requested_models = None
+
+    passed = True
+    if TOKEN_HUB is None:
+        passed = False
+    else:
+        print("Pulling evaluation requests and results.")
+
+        eval_queue_repo = Repository(
+            local_dir=QUEUE_PATH,
+            clone_from=QUEUE_REPO,
+            use_auth_token=TOKEN_HUB,
+            repo_type="dataset",
+        )
+        eval_queue_repo.git_pull()
+        
+        # Local directory where dataset repo is cloned + folder with eval requests
+        directory = QUEUE_PATH / EVAL_REQUESTS_PATH
+        requested_models = get_all_requested_models(directory)
+        requested_models = [p.stem for p in requested_models]
+        # Local directory where dataset repo is cloned
+        csv_results = get_csv_with_results(QUEUE_PATH)
+        if csv_results is None:
+            passed = False
+    if not passed:
+        raise ValueError("No Hugging Face token provided. Skipping evaluation requests and results.")
+
+    return eval_queue_repo, requested_models, csv_results
+
+def upload_file(requested_model_name, path_or_fileobj):
+    dest_repo_file = Path(EVAL_REQUESTS_PATH) / path_or_fileobj.name
+    dest_repo_file = str(dest_repo_file)
+    hf_api.upload_file(
+            path_or_fileobj=path_or_fileobj,
+            path_in_repo=str(dest_repo_file),
+            repo_id=QUEUE_REPO,
+            token=TOKEN_HUB,
+            repo_type="dataset",
+            commit_message=f"Add {requested_model_name} to eval queue")
+
+def get_all_requested_models(directory):
+    directory = Path(directory)
+    all_requested_models = list(directory.glob("*.txt"))
+    return all_requested_models
+
+def get_csv_with_results(directory):
+    directory = Path(directory)
+    all_csv_files = list(directory.glob("*.csv"))
+    latest = [f for f in all_csv_files if f.stem.endswith("latest")]
+    if len(latest) != 1:
+        return None
+    return latest[0]
+
+def is_model_on_hub(model_name, revision="main"):
+    try:
+        model_name = model_name.replace(" ","")
+        author = model_name.split("/")[0]
+        model_id = model_name.split("/")[1]
+        if len(author) == 0 or len(model_id) == 0:
+            return False, "is not a valid model name. Please use the format `author/model_name`."
+    except Exception as e:
+        return False, "is not a valid model name. Please use the format `author/model_name`."
+
+    try:
+        models = list(hf_api.list_models(author=author, search=model_id))
+        matched = [model_name for m in models if m.modelId == model_name]
+        if len(matched) != 1:
+            return False, "was not found on the hub!"
+        else:
+            return True, None
+    except Exception as e:
+        print(f"Could not get the model from the hub.: {e}")
+        return False, "was not found on hub!"

src/phoneme_eval.py → phoneme_eval.py

@@ -1,7 +1,7 @@
 import pandas as pd
-from src.utils.load_model import run_hubert_base, run_whisper, run_model, run_timit
-from src.utils.audio_process import calculate_error_rate, load_audio
-from src.utils.cmu_process import clean_cmu, cmu_to_ipa
+from utils.load_model import run_hubert_base, run_whisper, run_model, run_timit
+from utils.audio_process import calculate_error_rate, load_audio
+from utils.cmu_process import clean_cmu, cmu_to_ipa
 
 def set_output(model, pre_pho, ref_pho, duration, per, score):
     return {
@@ -16,7 +16,7 @@ def set_output(model, pre_pho, ref_pho, duration, per, score):
 # Map model names to their runner functions
 MODEL_RUNNERS = {
     "HuBERT-Base": run_hubert_base,
-    # "Whisper": run_whisper,
+    "Whisper": run_whisper,
     "HuBERT fine-tuned": run_model,
     "Timit": run_timit
 }
@@ -47,7 +47,7 @@ def benchmark_all(example):
     # Run all models
     results = [
         get_output("HuBERT-Base", wav, reference_phoneme),
-        # get_output("Whisper", wav, reference_phoneme),
+        get_output("Whisper", wav, reference_phoneme),
         get_output("HuBERT fine-tuned", wav, reference_phoneme),
         get_output("Timit", wav, reference_phoneme),
     ]
@@ -133,12 +133,13 @@ def main():
 
     # Save results for leaderboard consumption (one JSON per model)
     import json, os, time
-    results_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "eval-results")
+    # results_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "eval-results")
+    results_dir = os.path.join("eval-results")
     os.makedirs(results_dir, exist_ok=True)
 
     timestamp = int(time.time())
     for model_name, task_results in per_model_results.items():
-        org_model = f"local/{model_name}"
+        org_model = f"{model_name}"
         payload = {
             "config": {
                 "model_name": org_model,

pyproject.toml

@@ -1,13 +0,0 @@
-[tool.ruff]
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
-line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
-
-[tool.isort]
-profile = "black"
-line_length = 119
-
-[tool.black]
-line-length = 119

requirements.txt

@@ -1,21 +1,13 @@
-APScheduler
-black
-datasets
 gradio
-gradio[oauth]
-gradio_leaderboard==0.0.13
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
 pandas
-python-dateutil
-tqdm
+numpy
 transformers
-tokenizers>=0.15.0
-sentencepiece
 torch
+torchaudio
+datasets
+huggingface-hub
+soundfile
+librosa
 nltk
 g2p-en
-librosa
-soundfile
+python-dotenv

src/about.py

@@ -1,74 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the results json, metric_key, column name for display
-    # Using actual dataset names as keys
-    phoneme_asr = Task("phoneme_asr", "per", "PER phoneme_asr")
-    kids_phoneme_md = Task("kids_phoneme_md", "per", "PER kids_phoneme_md")
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Phoneme Detection Leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-This leaderboard ranks phoneme detection models by average PER (lower is better).
-Evaluations aggregate across phoneme_asr and kids_phoneme_md datasets for a fair comparison.
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-We compute Phoneme Error Rate (PER) per dataset/split and aggregate an average.
-
-## Reproducibility
-Ensure your model and tokenizer can be loaded via Transformers AutoClasses.
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""

src/display/css_html_js.py

@@ -1,105 +0,0 @@
-custom_css = """
-
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#models-to-add-text {
-    font-size: 18px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#leaderboard-table-lite {
-    margin-top: 15px
-}
-
-#search-bar-table-box > div:first-child {
-    background: none;
-    border: none;
-}
- 
-#search-bar {
-    padding: 0px;
-}
-
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-#leaderboard-table td:nth-child(2),
-#leaderboard-table th:nth-child(2) {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
-}
-
-#scale-logo .download {
-    display: none;
-}
-#filter_type{
-    border: 0;
-    padding-left: 0;
-    padding-top: 0;
-}
-#filter_type label {
-    display: flex;
-}
-#filter_type label > span{
-    margin-top: var(--spacing-lg);
-    margin-right: 0.5em;
-}
-#filter_type label > .wrap{
-    width: 103px;
-}
-#filter_type label > .wrap .wrap-inner{  
-    padding: 2px;
-}
-#filter_type label > .wrap .wrap-inner input{
-    width: 1px
-}
-#filter-columns-type{
-    border:0;
-    padding:0.5;
-}
-#filter-columns-size{
-    border:0;
-    padding:0.5;
-}
-#box-filter > .form{
-    border: 0
-}
-"""
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-import pandas as pd
-
-from src.about import Tasks  # assume Tasks = [Task1, Task2, ...]
-
-def fields(raw_class):
-    return [
-        v for k, v in raw_class.__dict__.items()
-        if not (k.startswith("__") and k.endswith("__"))
-    ]
-
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-# -------------------------------------------------------------------
-# Build leaderboard columns
-# -------------------------------------------------------------------
-auto_eval_column_dict = []
-
-# Rank/Model/Badge
-auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent("Rank", "number", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-auto_eval_column_dict.append(["badge", ColumnContent, ColumnContent("Badge", "str", True)])
-
-# Per-dataset metrics
-# Example: "PER ⬇️ (TIMIT)", "Avg Duration (s) (TIMIT)"
-for task in Tasks:
-    dataset_name = task.name   # short name
-    col_base = task.value.col_name  # e.g. "PER ⬇️"
-    # allow multiple metrics per dataset if needed
-    auto_eval_column_dict.append([
-        f"{dataset_name}_per",
-        ColumnContent,
-        ColumnContent(f"{col_base} ({dataset_name})", "number", True),
-    ])
-    auto_eval_column_dict.append([
-        f"{dataset_name}_avg_duration",
-        ColumnContent,
-        ColumnContent(f"Avg Duration (s) ({dataset_name})", "number", True),
-    ])
-
-# Global average across datasets
-auto_eval_column_dict.append([
-    "average", ColumnContent, ColumnContent("Avg PER ⬇️ (All)", "number", True)
-])
-
-# Extra model info
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# Final dataclass
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-# -------------------------------------------------------------------
-# Example: Create dataframe header
-# -------------------------------------------------------------------
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-df = pd.DataFrame(columns=[c.name for c in fields(AutoEvalColumn)])

src/envs.py

@@ -1,25 +0,0 @@
-import os
-
-from huggingface_hub import HfApi
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
-
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,207 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn
-from src.about import Tasks
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: str = "Unknown"
-    model_type: str = "Unknown" # Pretrained, fine tuned, ...
-    weight_type: str = "Original" # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Precision
-        precision = str(config.get("model_dtype", "Unknown"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            per_vals = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if per_vals.size > 0 and not any([val is None for val in per_vals]):
-                results[f"{task.benchmark}_per"] = float(np.mean(per_vals))
-
-            # Average duration if present
-            dur_vals = np.array([v.get("avg_duration", None) for k, v in data["results"].items() if task.benchmark == k])
-            if dur_vals.size > 0 and not any([val is None for val in dur_vals]):
-                results[f"{task.benchmark}_avg_duration"] = float(np.mean(dur_vals))
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = str(request.get("model_type", "Unknown"))
-            self.weight_type = str(request.get("weight_type", "Original"))
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        # Compute average PER across tasks from per-keys only
-        per_values = [v for k, v in self.results.items() if k.endswith("_per") and v is not None]
-        average = sum(per_values) / len(per_values) if per_values else None
-        data_dict = {
-            AutoEvalColumn.rank.name: None,
-            AutoEvalColumn.badge.name: "",
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision,
-            AutoEvalColumn.model_type.name: self.model_type,
-            AutoEvalColumn.weight_type.name: self.weight_type,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            dataset = task.name
-            # Use display labels matching utils.AutoEvalColumn definitions
-            per_label = f"{task.value.col_name} ({dataset})"
-            dur_label = f"Avg Duration (s) ({dataset})"
-            data_dict[per_label] = self.results.get(f"{task.value.benchmark}_per")
-            data_dict[dur_label] = self.results.get(f"{task.value.benchmark}_avg_duration")
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/populate.py

@@ -1,63 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    # If no data yet, return an empty DataFrame with expected columns
-    if df.empty or AutoEvalColumn.average.name not in df.columns:
-        return pd.DataFrame(columns=cols)
-
-    # Lower PER is better: sort ascending
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=True)
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data["Model"] = make_clickable_model(data["model"])
-            data["Model sha"] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data["Model"] = make_clickable_model(data["model"])
-                data["Model sha"] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )

test_basic.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""
+Basic test to verify the cleaned up phoneme detection leaderboard functionality.
+"""
+
+import os
+import sys
+import json
+import tempfile
+import pandas as pd
+
+# Add current directory to path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+def test_imports():
+    """Test that all modules can be imported"""
+    try:
+        from constants import BANNER, INTRODUCTION_TEXT
+        from utils_display import PhonemeEvalColumn, make_clickable_model
+        from init import is_model_on_hub
+        print("All imports successful")
+        return True
+    except ImportError as e:
+        print(f"Import error: {e}")
+        return False
+
+def test_data_loading():
+    """Test that the app can load data from eval-results directory"""
+    try:
+        from app import load_results, EVAL_RESULTS_DIR
+        
+        # Create a temporary test result
+        os.makedirs(EVAL_RESULTS_DIR, exist_ok=True)
+        test_result = {
+            "config": {
+                "model_name": "test/model",
+                "model_dtype": "float32",
+                "model_sha": "test123"
+            },
+            "results": {
+                "phoneme_asr": {"per": 15.5, "avg_duration": 0.1},
+                "kids_phoneme_md": {"per": 18.2, "avg_duration": 0.12}
+            }
+        }
+        
+        test_file = os.path.join(EVAL_RESULTS_DIR, "test_results.json")
+        with open(test_file, "w") as f:
+            json.dump(test_result, f)
+        
+        # Test loading
+        df = load_results(EVAL_RESULTS_DIR)
+        print(f"Data loading successful, found {len(df)} rows")
+        
+        # Clean up
+        os.remove(test_file)
+        return True
+        
+    except Exception as e:
+        print(f"Data loading error: {e}")
+        return False
+
+def test_utils():
+    """Test utility functions"""
+    try:
+        from utils_display import make_clickable_model, styled_error, styled_message
+        
+        # Test model link generation
+        link = make_clickable_model("facebook/hubert-base")
+        assert "facebook/hubert-base" in link
+        assert "href=" in link
+        
+        # Test styled messages
+        error_msg = styled_error("Test error")
+        assert "red" in error_msg
+        
+        success_msg = styled_message("Test success")
+        assert "green" in success_msg
+        
+        print("Utility functions working")
+        return True
+        
+    except Exception as e:
+        print(f"Utility test error: {e}")
+        return False
+
+def main():
+    """Run all tests"""
+    print("Testing Phoneme Detection Leaderboard...")
+    
+    tests = [
+        test_imports,
+        test_data_loading,
+        test_utils
+    ]
+    
+    passed = 0
+    total = len(tests)
+    
+    for test in tests:
+        if test():
+            passed += 1
+        print()
+    
+    print(f"Test Results: {passed}/{total} tests passed")
+    
+    if passed == total:
+        print("All tests passed! The cleaned up version is working correctly.")
+        return True
+    else:
+        print("Some tests failed. Please check the errors above.")
+        return False
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

utils/__init__.py

@@ -0,0 +1 @@
+# Utils package for phoneme detection leaderboard

{src/utils → utils}/audio_process.py

@@ -164,4 +164,4 @@ def calculate_error_rate(ref_seq, hyp_seq, unit="phoneme"):
     N = len(ref_seq)  # reference length
     error_rate = (S + D + I) / N if N > 0 else 0.0
 
-    return error_rate*100, {"S": S, "D": D, "I": I, "N": N}
+    return error_rate*100, {"S": S, "D": D, "I": I, "N": N}

{src/utils → utils}/cmu_process.py

@@ -108,4 +108,4 @@ def text_to_phoneme(text):
     phonemes = safe_g2p(clean_text(text))
     res = "".join(phonemes)
     res = clean_cmu(res)
-    return res
+    return res

{src/utils → utils}/load_model.py

@@ -39,6 +39,7 @@ whisper_model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-
 
 # 3. My Hubert Model (optional HF token via env)
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
+# print(HF_TOKEN)
 proc = Wav2Vec2Processor.from_pretrained("tecasoftai/hubert-finetune", token=HF_TOKEN)
 model = HubertForCTC.from_pretrained("tecasoftai/hubert-finetune", token=HF_TOKEN).to(device).eval()
 
@@ -114,4 +115,4 @@ def run_timit(wav):
     phonemes = timit_proc.batch_decode(predicted_ids)
     phonemes = "".join(phonemes)
     
-    return phonemes.strip(), time.time() - start
+    return phonemes.strip(), time.time() - start

utils_display.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class PhonemeEvalColumn: # Phoneme evals column
+    model = ColumnContent("Model", "markdown")
+    avg_per = ColumnContent("Average PER ⬇️", "number")    
+    avg_duration = ColumnContent("Avg Duration (s)", "number")
+    per_phoneme_asr = ColumnContent("PER phoneme_asr", "number")
+    per_kids_phoneme_md = ColumnContent("PER kids_phoneme_md", "number")
+
+def make_clickable_model(model_name):
+    model_name_list = model_name.split("/")
+    if model_name_list[0] == "local":
+        link = "#"  # Local models don't have external links
+    elif model_name_list[0] == "facebook":
+        link = f"https://huggingface.co/{model_name}"
+    elif model_name_list[0] == "openai":
+        link = "https://openai.com/"
+    elif model_name_list[0] == "HuBERT-Base":
+        link = "https://huggingface.co/facebook/hubert-base-ls960"
+    elif model_name_list[0] == "HuBERT-fine-tuned":
+        link = "https://huggingface.co/tecasoftai/hubert-finetune"
+    elif model_name_list[0] == "Timit":
+        link = "https://huggingface.co/vitouphy/wav2vec2-xls-r-300m-timit-phoneme"
+    elif model_name_list[0] == "Whisper":
+        link = "https://huggingface.co/openai/whisper-base"
+    else:
+        link = f"https://huggingface.co/{model_name}"
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"