update leaderboard results

app.py

@@ -10,14 +10,15 @@ from src.about import (
 )
 from src.display.css_html_js import custom_css
 from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    AutoEvalColumn,
+    CATEGORY_ACCURACY_COLS,
+    CATEGORY_F1_COLS,
+    OVERALL_TIER_COLS,
+    CategoryAccuracyColumn,
+    CategoryF1Column,
+    OverallTierColumn,
     fields,
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
 from src.populate import get_leaderboard_df
 
 
@@ -25,31 +26,51 @@ def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
-LEADERBOARD_DF = get_leaderboard_df(
-    EVAL_RESULTS_PATH + "/ARFBench_leaderboard.csv", EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS
+OVERALL_TIER_LEADERBOARD_DF = get_leaderboard_df(
+    EVAL_RESULTS_PATH + "/ARFBench_leaderboard.csv",
+    EVAL_REQUESTS_PATH,
+    OVERALL_TIER_COLS,
+    OVERALL_TIER_COLS,
+    sort_by="overall_f1",
 )
 
+CATEGORY_F1_LEADERBOARD_DF = get_leaderboard_df(
+    EVAL_RESULTS_PATH + "/ARFBench_leaderboard_category_f1.csv",
+    EVAL_REQUESTS_PATH,
+    CATEGORY_F1_COLS,
+    CATEGORY_F1_COLS,
+    sort_by="overall_f1",
+)
+
+CATEGORY_ACCURACY_LEADERBOARD_DF = get_leaderboard_df(
+    EVAL_RESULTS_PATH + "/ARFBench_leaderboard_category_accuracy.csv",
+    EVAL_REQUESTS_PATH,
+    CATEGORY_ACCURACY_COLS,
+    CATEGORY_ACCURACY_COLS,
+    sort_by="overall_accuracy",
+)
 
-def init_leaderboard(dataframe):
+
+def init_custom_leaderboard(dataframe, column_class, filter_column_name, filter_label):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
     return Leaderboard(
         value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
+        datatype=[c.type for c in fields(column_class)],
         select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            default_selection=[c.name for c in fields(column_class) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(column_class) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        search_columns=[column_class.model.name],
+        hide_columns=[c.name for c in fields(column_class) if c.hidden],
         filter_columns=[
             ColumnFilter(
-                AutoEvalColumn.pass_at_1.name,
+                filter_column_name,
                 type="slider",
                 min=0,
                 max=100,
-                label="pass@1 score",
+                label=filter_label,
             ),
         ],
         bool_checkboxgroup_label="Hide models",
@@ -64,7 +85,30 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 ARFBench Leaderboard", elem_id="arfbench-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+            with gr.Tabs(selected=0):
+                with gr.TabItem("Overall + Tier (Default)", id=0):
+                    leaderboard_overall_tier = init_custom_leaderboard(
+                        OVERALL_TIER_LEADERBOARD_DF,
+                        OverallTierColumn,
+                        OverallTierColumn.overall_f1.name,
+                        "Overall F1 score",
+                    )
+
+                with gr.TabItem("Per-Category F1", id=1):
+                    leaderboard_category_f1 = init_custom_leaderboard(
+                        CATEGORY_F1_LEADERBOARD_DF,
+                        CategoryF1Column,
+                        CategoryF1Column.overall_f1.name,
+                        "Overall F1 score",
+                    )
+
+                with gr.TabItem("Per-Category Accuracy", id=2):
+                    leaderboard_category_accuracy = init_custom_leaderboard(
+                        CATEGORY_ACCURACY_LEADERBOARD_DF,
+                        CategoryAccuracyColumn,
+                        CategoryAccuracyColumn.overall_accuracy.name,
+                        "Overall Accuracy score",
+                    )
 
         with gr.TabItem("📝 About", elem_id="about-tab-table", id=1):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

results/ARFBench_leaderboard.csv

@@ -1,14 +1,22 @@
-Model,pass@1,pass@5,Presence,Identification,Start Time,End Time,Magnitude,Categorization,Correlation,Indicator
-Random Choice,23.5,-,50.0,12.5,18.2,16.7,12.5,16.7,20.0,20.0
-Frequent Choice,46.9,-,82.9,36.8,21.4,31.3,26.3,30.8,82.9,31.7
-Oracle GPT-4o,57.5,-,87.4,34.2,26.8,25.0,55.3,62.5,82.9,28.6
-GPT-4.1,57.6,62.2,82.9,39.5,39.2,37.5,60.5,54.8,72.9,34.9
-Claude 3.7 Sonnet,56.7,57.4,85.6,34.2,41.0,40.6,53.9,54.8,67.1,36.5
-GPT-4o,54.4,60.2,82.0,28.9,23.2,34.4,52.6,52.9,80.0,34.9
-o4-mini,48.5,64.5,80.2,13.2,33.9,43.8,48.7,47.1,57.1,22.2
-InternVL3-78B,43.5,47.2,84.7,31.6,32.1,31.3,30.3,47.1,24.3,25.4
-Qwen2.5-VL-72B,41.1,53.8,83.8,21.1,25.0,6.3,32.9,26.0,48.6,36.5
-LlaVa-OneVision-72B,38.9,44.6,76.6,26.3,23.2,15.6,21.1,42.3,42.9,17.5
-Llama-3.2-Vision-90B,34.9,37.8,76.6,13.2,26.8,21.8,28.9,21.2,35.7,17.5
-QvQ,19.8,22.9,11.7,15.8,1.79,0.00,27.6,36.5,28.6,15.9
-ChatTS,10.0,10.4,3.60,10.5,14.3,3.13,9.21,14.4,18.6,4.76
+Model,Model Type,Accuracy,Tier I Accuracy,Tier II Accuracy,Tier III Accuracy,Overall F1,Tier I F1,Tier II F1,Tier III F1
+Random Choice,Baseline,24.5,50.0,20.0,20.0,22.5,45.6,20.5,16.8
+Per-category Frequent Choice,Baseline,45.1,84.7,30.1,45.6,17.3,45.9,12.3,12.5
+Non-domain Experts (n=2),Baseline,69.7,80.4,63.2,72.0,60.7,68.0,59.9,59.0
+Domain Experts (n=2),Baseline,72.7,89.3,67.7,71.4,64.6,76.1,64.5,60.9
+Model-Expert Oracle,Baseline,87.2,96.4,80.3,90.5,82.8,89.0,77.1,86.3
+Qwen3 32B,LLM,47.9,80.9,35.1,48.6,36.1,55.7,31.5,33.8
+GPT-5 (text),LLM,56.4,82.6,45.2,57.9,43.8,66.1,39.6,40.3
+Qwen3-VL 8B,VLM,45.3,80.2,40.8,37.8,34.7,63.5,36.1,23.6
+Claude Sonnet 4.5,VLM,47.2,83.8,43.5,38.4,37.9,63.2,40.6,26.9
+GPT-4o,VLM,47.2,79.3,49.0,34.8,42.4,64.2,43.8,33.8
+GPT-4.1,VLM,47.9,80.2,50.3,34.8,44.0,65.1,48.0,33.1
+Qwen3-VL 32B,VLM,52.8,80.2,46.7,49.2,45.1,65.1,41.9,41.3
+Claude Opus 4.6,VLM,54.8,88.3,52.3,45.9,46.7,65.8,49.1,38.2
+Gemini 3 Pro,VLM,58.1,82.9,51.0,56.5,49.6,67.8,49.7,43.4
+GPT-5.4,VLM,61.3,81.1,54.2,61.3,51.4,62.6,50.4,48.4
+GPT-5,VLM,62.7,82.0,55.9,62.5,51.9,66.9,51.2,47.5
+OpenTSLM 1B (TS-LLM),Post-trained TSFM,0.8,0.0,2.0,0.0,1.2,0.0,3.0,0.0
+ChatTS 8B (TS-LLM),Post-trained TSFM,31.1,60.4,26.5,25.5,22.1,48.1,20.0,15.4
+Toto-Qwen3 32B (TSFM-LLM),Post-trained TSFM,48.8,82.9,47.4,38.7,33.9,60.0,43.6,16.4
+Qwen3-VL 32B (post-trained),Post-trained TSFM,56.9,84.7,50.3,53.8,46.6,69.8,44.9,40.5
+Toto-VLM 32B (TSFM-VLM),Post-trained TSFM,63.9,84.7,55.6,64.6,48.9,66.3,48.4,43.5

results/ARFBench_leaderboard_category_accuracy.csv

@@ -0,0 +1,22 @@
+Model,Model Type,Overall Accuracy,Presence,Identification,Start Time,End Time,Magnitude,Categorization,Correlation,Indicator
+Random Choice,Baseline,24.5,50.0,20.0,20.0,20.0,20.0,20.0,20.0,20.0
+Per-category Frequent Choice,Baseline,45.1,84.7,36.8,35.7,34.4,17.1,32.7,42.9,48.5
+Non-domain Experts (n=2),Baseline,69.7,80.4,66.7,64.3,68.8,60.5,61.5,72.1,72.0
+Domain Experts (n=2),Baseline,72.7,89.3,77.8,67.9,75.0,60.5,72.4,74.4,68.3
+Model-Expert Oracle,Baseline,87.2,96.4,77.8,78.6,100.0,68.4,84.6,95.4,85.4
+Qwen3 32B (text),LLM,47.9,80.9,28.9,27.3,35.5,37.3,39.8,50.9,46.3
+GPT-5 (text),LLM,56.4,82.6,47.4,29.6,38.7,51.4,50.0,56.9,59.0
+Qwen3-VL 8B,VLM,45.3,80.2,26.3,25.0,31.3,57.9,45.2,57.1,17.8
+Claude Sonnet 4.5,VLM,47.2,83.8,18.4,30.4,37.5,53.9,53.8,58.8,17.2
+GPT-4o,VLM,47.2,79.3,39.5,35.7,43.8,61.8,51.9,45.3,23.9
+GPT-4.1,VLM,47.9,80.2,28.9,33.9,40.6,68.4,56.7,45.9,23.3
+Qwen3-VL 32B,VLM,52.8,80.2,23.7,33.9,56.3,59.2,50.0,61.8,36.2
+Claude Opus 4.6,VLM,54.8,88.3,31.6,37.5,53.1,57.9,63.5,65.9,25.2
+Gemini 3 Pro,VLM,58.1,82.9,28.9,44.6,62.5,56.7,54.8,71.2,41.1
+GPT-5.4,VLM,61.3,81.1,31.6,63.6,65.6,57.9,56.7,61.8,60.7
+GPT-5,VLM,62.7,82.0,31.6,44.6,68.8,65.8,59.6,63.5,61.3
+OpenTSLM (TS-LLM),Post-trained TSFM,0.8,0.0,0.0,3.6,0.0,5.3,0.0,0.0,0.0
+ChatTS (TS-LLM),Post-trained TSFM,31.1,59.5,15.8,16.1,15.6,28.9,20.2,40.0,14.7
+Toto-Qwen3 (TSFM-LLM),Post-trained TSFM,48.8,82.9,10.5,35.7,34.4,47.4,71.2,41.8,35.6
+Qwen3-VL 32B (post-trained),Post-trained TSFM,56.9,84.7,36.8,41.1,43.8,63.2,52.9,67.6,39.3
+Toto-VLM (TSFM-VLM),Post-trained TSFM,63.9,84.7,47.4,26.8,59.4,64.5,66.3,68.8,60.1

results/ARFBench_leaderboard_category_f1.csv

@@ -0,0 +1,22 @@
+Model,Model Type,Overall F1,Presence,Identification,Start Time,End Time,Magnitude,Categorization,Correlation,Indicator
+Random Choice,Baseline,22.5,45.6,21.2,18.9,18.2,20.4,21.7,15.8,17.8
+Per-category Frequent Choice,Baseline,17.3,45.9,10.8,16.3,14.1,6.0,14.6,12.0,13.1
+Non-domain Experts (n=2),Baseline,61.3,68.0,79.0,67.4,67.2,40.3,61.2,58.4,62.4
+Domain Experts (n=2),Baseline,64.6,76.1,77.5,74.2,72.6,51.8,67.3,64.1,57.6
+Model-Expert Oracle,Baseline,82.8,89.0,68.3,83.4,1.0,67.0,75.6,94.4,77.8
+Qwen3 32B,LLM,36.1,55.7,28.4,26.6,26.9,31.4,36.8,32.3,35.4
+GPT-5 (text),LLM,43.8,66.1,38.1,27.9,27.0,44.8,47.6,38.0,42.4
+Qwen3-VL 8B,VLM,34.7,63.5,28.6,21.8,23.5,47.0,42.8,33.1,13.8
+Claude Sonnet 4.5,VLM,37.9,63.2,16.8,33.2,31.3,49.3,49.8,33.8,19.8
+GPT-4o,VLM,42.4,64.2,34.6,30.3,36.1,51.8,50.8,40.1,27.2
+GPT-4.1,VLM,44.0,65.1,29.2,33.5,32.7,63.7,55.9,42.9,23.3
+Qwen3-VL 32B,VLM,45.1,65.1,25.0,30.8,46.7,46.9,49.0,47.5,34.7
+Claude Opus 4.6,VLM,46.7,65.8,34.3,36.1,45.1,53.8,59.2,51.6,24.1
+Gemini 3 Pro,VLM,49.6,67.8,38.6,43.3,57.1,50.3,54.5,57.0,29.2
+GPT-5.4,VLM,51.4,62.6,29.6,53.3,55.1,51.7,54.1,47.7,49.1
+GPT-5,VLM,51.9,66.8,32.8,44.2,47.8,59.1,57.0,49.0,45.9
+OpenTSLM 1B (TS-LLM),Post-trained TSFM,1.2,0.0,8.2,2.7,0.0,6.0,0.0,0.0,0.0
+ChatTS 8B (TS-LLM),Post-trained TSFM,22.1,48.1,22.2,15.0,14.4,27.9,17.9,21.4,9.2
+Toto-Qwen3 32B (TSFM-LLM),Post-trained TSFM,33.9,59.9,17.5,41.3,23.0,35.9,66.2,18.6,14.1
+Qwen3-VL 32B (post-trained),Post-trained TSFM,46.6,69.7,40.5,37.2,36.7,48.9,50.3,46.8,33.9
+Toto-VLM 32B (TSFM-VLM),Post-trained TSFM,48.9,66.3,46.9,23.0,48.8,54.1,58.4,44.2,42.7

src/about.py

@@ -27,7 +27,7 @@ TITLE = """<h1 align="center" id="space-title">ARFBench Multimodal Time Series R
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
 **ARF**Bench (**A**nomaly **R**easoning **F**ramework Benchmark) is a
-multimodal time-series reasoning benchmark consisting of 550 question-answer
+multimodal time-series reasoning benchmark consisting of 750 question-answer
 (QA) pairs composed from real-world incident data collected at Datadog,
 a leading observability platform.
 

src/display/formatting.py

@@ -1,24 +1,41 @@
 def model_hyperlink(link, model_name):
     if model_name == "":
         return model_name
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+    style = (
+        "color: var(--link-text-color); "
+        "text-decoration: underline;"
+        "text-decoration-style: dotted;"
+    )
+    return f'<a target="_blank" href="{link}" style="{style}">{model_name}</a>'
 
 
 def make_clickable_model(model_name):
+    if not isinstance(model_name, str):
+        model_name = str(model_name)
+
+    model_name = model_name.strip()
+
+    # Only convert valid Hugging Face repository paths (org/model) into links.
+    if "/" not in model_name or " " in model_name:
+        return model_name
+
     link = f"https://huggingface.co/{model_name}"
     return model_hyperlink(link, model_name)
 
 
 def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+    style = "color: red; font-size: 20px; text-align: center;"
+    return f"<p style='{style}'>{error}</p>"
 
 
 def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+    style = "color: orange; font-size: 20px; text-align: center;"
+    return f"<p style='{style}'>{warn}</p>"
 
 
 def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+    style = "color: green; font-size: 20px; text-align: center;"
+    return f"<p style='{style}'>{message}</p>"
 
 
 def has_no_nan_values(df, columns):

src/display/utils.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass, make_dataclass
+from dataclasses import dataclass
 from enum import Enum
 
 
@@ -19,27 +19,76 @@ class ColumnContent:
 
 
 # ARFBench Leaderboard columns
-auto_eval_column_dict = []
-# Model column (always displayed)
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-# Performance metrics
-auto_eval_column_dict.append(["pass_at_1", ColumnContent, ColumnContent("pass@1", "number", True)])
-auto_eval_column_dict.append(["pass_at_5", ColumnContent, ColumnContent("pass@5", "number", True)])
-# Specific benchmark metrics
-auto_eval_column_dict.append(["presence", ColumnContent, ColumnContent("Presence", "number", True)])
-auto_eval_column_dict.append(["identification", ColumnContent, ColumnContent("Identification", "number", True)])
-auto_eval_column_dict.append(["start_time", ColumnContent, ColumnContent("Start Time", "number", True)])
-auto_eval_column_dict.append(["end_time", ColumnContent, ColumnContent("End Time", "number", True)])
-auto_eval_column_dict.append(["magnitude", ColumnContent, ColumnContent("Magnitude", "number", True)])
-auto_eval_column_dict.append(["categorization", ColumnContent, ColumnContent("Categorization", "number", True)])
-auto_eval_column_dict.append(["correlation", ColumnContent, ColumnContent("Correlation", "number", True)])
-auto_eval_column_dict.append(["indicator", ColumnContent, ColumnContent("Indicator", "number", True)])
-
-# We use make dataclass to dynamically fill the scores
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-
-## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class AutoEvalColumn:
+    # Model column (always displayed)
+    model = ColumnContent("model", "markdown", True, never_hidden=True)
+    # Model type column
+    model_type = ColumnContent("model_type", "str", True)
+    # Performance metrics
+    overall_f1 = ColumnContent("overall_f1", "number", True)
+    tier_i_f1 = ColumnContent("tier_i_f1", "number", True)
+    tier_ii_f1 = ColumnContent("tier_ii_f1", "number", True)
+    tier_iii_f1 = ColumnContent("tier_iii_f1", "number", True)
+    # Specific benchmark metrics
+    presence = ColumnContent("presence", "number", True)
+    identification = ColumnContent("identification", "number", True)
+    start_time = ColumnContent("start_time", "number", True)
+    end_time = ColumnContent("end_time", "number", True)
+    magnitude = ColumnContent("magnitude", "number", True)
+    categorization = ColumnContent("categorization", "number", True)
+    correlation = ColumnContent("correlation", "number", True)
+    indicator = ColumnContent("indicator", "number", True)
+
+
+# Overall + per-tier leaderboard columns
+@dataclass(frozen=True)
+class OverallTierColumn:
+    model = ColumnContent("model", "markdown", True, never_hidden=True)
+    model_type = ColumnContent("model_type", "str", True)
+    accuracy = ColumnContent("accuracy", "number", True)
+    tier_i_accuracy = ColumnContent("tier_i_accuracy", "number", True)
+    tier_ii_accuracy = ColumnContent("tier_ii_accuracy", "number", True)
+    tier_iii_accuracy = ColumnContent("tier_iii_accuracy", "number", True)
+    overall_f1 = ColumnContent("overall_f1", "number", True)
+    tier_i_f1 = ColumnContent("tier_i_f1", "number", True)
+    tier_ii_f1 = ColumnContent("tier_ii_f1", "number", True)
+    tier_iii_f1 = ColumnContent("tier_iii_f1", "number", True)
+
+
+# Per-category F1 leaderboard columns
+@dataclass(frozen=True)
+class CategoryF1Column:
+    model = ColumnContent("model", "markdown", True, never_hidden=True)
+    model_type = ColumnContent("model_type", "str", True)
+    overall_f1 = ColumnContent("overall_f1", "number", True)
+    presence = ColumnContent("presence", "number", True)
+    identification = ColumnContent("identification", "number", True)
+    start_time = ColumnContent("start_time", "number", True)
+    end_time = ColumnContent("end_time", "number", True)
+    magnitude = ColumnContent("magnitude", "number", True)
+    categorization = ColumnContent("categorization", "number", True)
+    correlation = ColumnContent("correlation", "number", True)
+    indicator = ColumnContent("indicator", "number", True)
+
+
+# Per-category accuracy leaderboard columns
+@dataclass(frozen=True)
+class CategoryAccuracyColumn:
+    model = ColumnContent("model", "markdown", True, never_hidden=True)
+    model_type = ColumnContent("model_type", "str", True)
+    overall_accuracy = ColumnContent("overall_accuracy", "number", True)
+    presence = ColumnContent("presence", "number", True)
+    identification = ColumnContent("identification", "number", True)
+    start_time = ColumnContent("start_time", "number", True)
+    end_time = ColumnContent("end_time", "number", True)
+    magnitude = ColumnContent("magnitude", "number", True)
+    categorization = ColumnContent("categorization", "number", True)
+    correlation = ColumnContent("correlation", "number", True)
+    indicator = ColumnContent("indicator", "number", True)
+
+
+# For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
@@ -50,7 +99,7 @@ class EvalQueueColumn:  # Queue column
     status = ColumnContent("status", "str", True)
 
 
-## All the model information that we might need
+# All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
@@ -59,10 +108,9 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    OS_VLM = ModelDetails(name="open vision-language", symbol="🟢")
-    P_VLM = ModelDetails(name="proprietary vision-language", symbol="🔶")
-    TSFM = ModelDetails(name="time-series FM", symbol="⭕")
-    R = ModelDetails(name="reasoning", symbol="🟦")
+    LLM = ModelDetails(name="LLM", symbol="🟢")
+    VLM = ModelDetails(name="VLM", symbol="🔶")
+    TSFM = ModelDetails(name="Post-trained TSFM", symbol="⭕")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -70,13 +118,11 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "proprietary vision-language" in type or "🔶" in type:
-            return ModelType.P_VLM
-        if "open vision-language" in type or "🟢" in type:
-            return ModelType.OS_VLM
-        if "reasoning" in type or "🟦" in type:
-            return ModelType.R
-        if "time-series FM" in type or "⭕" in type:
+        if "VLM" in type or "🔶" in type:
+            return ModelType.VLM
+        if "LLM" in type or "🟢" in type:
+            return ModelType.LLM
+        if "TSFM" in type or "⭕" in type:
             return ModelType.TSFM
         return ModelType.Unknown
 
@@ -108,8 +154,12 @@ EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 # Define the benchmark columns for ARFBench
 BENCHMARK_COLS = [
-    "pass_at_1",
-    "pass_at_5",
+    "model",
+    "model_type",
+    "overall_f1",
+    "tier_i_f1",
+    "tier_ii_f1",
+    "tier_iii_f1",
     "presence",
     "identification",
     "start_time",
@@ -119,3 +169,9 @@ BENCHMARK_COLS = [
     "correlation",
     "indicator",
 ]
+
+
+# New leaderboard datasets
+OVERALL_TIER_COLS = [c.name for c in fields(OverallTierColumn) if not c.hidden]
+CATEGORY_F1_COLS = [c.name for c in fields(CategoryF1Column) if not c.hidden]
+CATEGORY_ACCURACY_COLS = [c.name for c in fields(CategoryAccuracyColumn) if not c.hidden]

src/populate.py

@@ -1,51 +1,48 @@
-import os
 import pandas as pd
 
 from src.display.formatting import make_clickable_model
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from the static CSV file"""
-    # Read the static CSV file
+def _normalize_columns(df: pd.DataFrame) -> pd.DataFrame:
+    rename_map = {
+        "model type": "model_type",
+        "overall f1": "overall_f1",
+        "overall accuracy": "overall_accuracy",
+        "accuracy": "accuracy",
+        "tier i": "tier_i_f1",
+        "tier ii": "tier_ii_f1",
+        "tier iii": "tier_iii_f1",
+        "tier i accuracy": "tier_i_accuracy",
+        "tier ii accuracy": "tier_ii_accuracy",
+        "tier iii accuracy": "tier_iii_accuracy",
+        "start time": "start_time",
+        "end time": "end_time",
+    }
+
+    normalized = {}
+    for col in df.columns:
+        cleaned = col.strip().lower()
+        normalized[col] = rename_map.get(cleaned, cleaned.replace(" ", "_"))
+
+    return df.rename(columns=normalized)
+
+
+def get_leaderboard_df(
+    results_path: str,
+    _requests_path: str,
+    _cols: list,
+    _benchmark_cols: list,
+    sort_by: str = "overall_f1",
+) -> pd.DataFrame:
+    """Creates a dataframe from a static CSV leaderboard file."""
     df = pd.read_csv(results_path)
+    df = _normalize_columns(df)
 
-    # Clean up column names to match our schema
-    df.columns = df.columns.str.replace("pass@1", "pass_at_1")
-    df.columns = df.columns.str.replace("pass@5", "pass_at_5")
-    df.columns = df.columns.str.replace("Start Time", "start_time")
-    df.columns = df.columns.str.replace("End Time", "end_time")
-    df.columns = df.columns.str.lower()
-
-    # Make model names clickable
-    df["model"] = df["model"].apply(make_clickable_model)
-
-    # Sort by pass@1 performance (descending)
-    df = df.sort_values(by=["pass_at_1"], ascending=False)
-
-    # Round numeric columns to 2 decimal places
-    numeric_cols = [
-        "pass_at_1",
-        "pass_at_5",
-        "presence",
-        "identification",
-        "start_time",
-        "end_time",
-        "magnitude",
-        "categorization",
-        "correlation",
-        "indicator",
-    ]
-
-    for col in numeric_cols:
-        if col in df.columns:
-            df[col] = df[col].round(2)
-
-    # Handle missing values - replace with 0 or appropriate value
-    df = df.fillna(0)
-
-    # Select only the columns we need
-    available_cols = [col for col in cols if col in df.columns]
-    df = df[available_cols]
+    if "model" in df.columns:
+        df["model"] = df["model"].apply(make_clickable_model)
+
+    if sort_by in df.columns:
+        df = df.sort_values(by=[sort_by], ascending=False)
 
     return df