app.py

import html
import json
import re
from pathlib import Path

import gradio as gr
import pandas as pd
from apscheduler.schedulers.background import BackgroundScheduler
from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
from huggingface_hub import snapshot_download

from src.about import (
    INTRODUCTION_TEXT,
    LLM_BENCHMARKS_TEXT,
    TITLE,
)
from src.display.css_html_js import custom_css, custom_js
from src.display.utils import (
    BENCHMARK_COLS,
    COLS,
    EVAL_COLS,
    AutoEvalColumn,
    fields,
)
from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
from src.populate import get_evaluation_queue_df, get_leaderboard_df


def restart_space():
    API.restart_space(repo_id=REPO_ID)


ENABLE_REMOTE_DATA_SYNC = False

if ENABLE_REMOTE_DATA_SYNC:
    try:
        print(EVAL_REQUESTS_PATH)
        snapshot_download(
            repo_id=QUEUE_REPO,
            local_dir=EVAL_REQUESTS_PATH,
            repo_type="dataset",
            tqdm_class=None,
            etag_timeout=30,
            token=TOKEN,
        )
    except Exception:
        restart_space()
    try:
        print(EVAL_RESULTS_PATH)
        snapshot_download(
            repo_id=RESULTS_REPO,
            local_dir=EVAL_RESULTS_PATH,
            repo_type="dataset",
            tqdm_class=None,
            etag_timeout=30,
            token=TOKEN,
        )
    except Exception:
        restart_space()

    LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)

    (
        finished_eval_queue_df,
        running_eval_queue_df,
        pending_eval_queue_df,
    ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)


def init_leaderboard(dataframe):
    if dataframe is None or dataframe.empty:
        raise ValueError("Leaderboard DataFrame is empty or None.")
    return Leaderboard(
        value=dataframe,
        datatype=[c.type for c in fields(AutoEvalColumn)],
        select_columns=SelectColumns(
            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
            label="Select Columns to Display:",
        ),
        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
        filter_columns=[
            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
            ColumnFilter(
                AutoEvalColumn.params.name,
                type="slider",
                min=0.01,
                max=150,
                label="Select the number of parameters (B)",
            ),
            ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
        ],
        bool_checkboxgroup_label="Hide models",
        interactive=False,
    )
def create_score_df(json_path: Path, decimals: int):
    data = json.loads(json_path.read_text(encoding="utf-8"))

    if not data:
        return pd.DataFrame()

    records = []
    for item in data:
        flat_record = {"ID": item.get("ID"), "Model": item.get("Model")}
        for category, sub_items in item.items():
            if category in ("ID", "Model"):
                continue
            if isinstance(sub_items, dict):
                cleaned_category = str(category).replace("\n", " ")
                for sub_category, value in sub_items.items():
                    header = f"{cleaned_category}\n{sub_category}"
                    flat_record[header] = value
        records.append(flat_record)

    df = pd.DataFrame(records)
    score_cols = [c for c in df.columns if c not in ("ID", "Model")]
    for col in score_cols:
        df[col] = df[col].apply(
            lambda v: ("" if pd.isna(v) else (f"{float(v):.{decimals}f}" if isinstance(v, (int, float)) else v))
        )
    return df


def dataframe_height(df: pd.DataFrame):
    rows = 0 if df is None else int(getattr(df, "shape", (0, 0))[0])
    row_px = 40
    header_px = 44
    padding_px = 96
    height = header_px + (rows * row_px) + padding_px
    return max(320, min(1200, height))


def create_gr_dataframe(value, **kwargs):
    dataframe_ctor = getattr(gr, "DataFrame", None) or getattr(gr, "Dataframe")
    remaining_kwargs = dict(kwargs)

    for _ in range(20):
        try:
            return dataframe_ctor(value, **remaining_kwargs)
        except TypeError as e:
            match = re.search(r"unexpected keyword argument '([^']+)'", str(e))
            if match is None:
                raise
            bad_key = match.group(1)
            if bad_key not in remaining_kwargs:
                raise
            remaining_kwargs.pop(bad_key)

    return dataframe_ctor(value, **remaining_kwargs)


def create_raw_score_df():
    raw_path = Path(__file__).resolve().parent / "src" / "raw_score.json"
    return create_score_df(raw_path, decimals=2)


def create_unweighted_z_score_df():
    z_path = Path(__file__).resolve().parent / "src" / "unweighted_z_score.json"
    return create_score_df(z_path, decimals=4)


def create_weighted_z_score_df():
    z_path = Path(__file__).resolve().parent / "src" / "weighted_z_score.json"
    data = json.loads(z_path.read_text(encoding="utf-8"))

    if not data:
        return pd.DataFrame()

    records = []
    for item in data:
        flat_record = {"ID": item.get("ID"), "Model": item.get("Model")}
        for category, sub_items in item.items():
            if category in ("ID", "Model"):
                continue
            if not isinstance(sub_items, dict):
                continue

            cleaned_category = str(category).replace("\n", " ")
            for sub_category, value in sub_items.items():
                if cleaned_category == "Overall Score":
                    header = "Overall Score"
                else:
                    header = f"{cleaned_category}\n{sub_category}"
                flat_record[header] = value
        records.append(flat_record)

    df = pd.DataFrame(records)

    overall_col = "Overall Score" if "Overall Score" in df.columns else None
    if overall_col is None:
        for c in df.columns:
            if isinstance(c, str) and c.endswith("\nOverall Score"):
                overall_col = c
                break

    if overall_col is not None:
        df[overall_col] = pd.to_numeric(df[overall_col], errors="coerce")
        df = df.sort_values(by=overall_col, ascending=False, kind="mergesort")

        cols = list(df.columns)
        fixed = [c for c in ("ID", "Model") if c in cols]
        rest = [c for c in cols if c not in set(fixed + [overall_col])]
        df = df[fixed + [overall_col] + rest]

    score_cols = [c for c in df.columns if c not in ("ID", "Model")]
    for col in score_cols:
        df[col] = pd.to_numeric(df[col], errors="coerce").apply(
            lambda v: "" if pd.isna(v) else f"{float(v) * 100:.2f}%"
        )

    if "ID" in df.columns:
        df["ID"] = pd.to_numeric(df["ID"], errors="coerce").astype("Int64")

    return df


def create_weights_table_html():
    weights_path = Path(__file__).resolve().parent / "src" / "weights.json"
    payload = json.loads(weights_path.read_text(encoding="utf-8"))
    bounds = payload["bounds"]
    min_row = bounds["min_row"]
    min_col = bounds["min_col"]
    rows = payload["rows"]

    covered = set()
    spans = {}
    for m in payload["merges"]:
        r1, c1, r2, c2 = m["r1"], m["c1"], m["r2"], m["c2"]
        spans[(r1, c1)] = {"rowspan": r2 - r1 + 1, "colspan": c2 - c1 + 1}
        for r in range(r1, r2 + 1):
            for c in range(c1, c2 + 1):
                if (r, c) != (r1, c1):
                    covered.add((r, c))

    parts = ['<div class="weights-scroll"><table class="weights-table">']
    for r_index, row in enumerate(rows, start=min_row):
        parts.append("<tr>")
        for c_index, value in enumerate(row, start=min_col):
            if (r_index, c_index) in covered:
                continue
            span = spans.get((r_index, c_index))
            attrs = ""
            if span is not None:
                attrs = f" rowspan=\"{span['rowspan']}\" colspan=\"{span['colspan']}\""
            tag = "th" if r_index == min_row else "td"
            if r_index != min_row:
                if isinstance(value, (int, float)):
                    value = f"{float(value):.4f}"
                elif isinstance(value, str):
                    stripped = value.strip()
                    try:
                        value = f"{float(stripped):.4f}"
                    except Exception:
                        pass
            text = "" if value is None else html.escape(str(value))
            parts.append(f"<{tag}{attrs}>{text}</{tag}>")
        parts.append("</tr>")
    parts.append("</table></div>")
    return "".join(parts)


RAW_SCORE_DF = create_raw_score_df()
UNWEIGHTED_Z_DF = create_unweighted_z_score_df()
WEIGHTED_Z_DF = create_weighted_z_score_df()

RAW_SCORE_COLUMN_WIDTHS = [40, 220] + [180] * (len(RAW_SCORE_DF.columns) - 2)
UNWEIGHTED_Z_COLUMN_WIDTHS = [40, 220] + [180] * (len(UNWEIGHTED_Z_DF.columns) - 2)
WEIGHTED_Z_COLUMN_WIDTHS = [40, 220] + [180] * (len(WEIGHTED_Z_DF.columns) - 2)

DATAFRAME_HEIGHT_CSS = f"""
#raw-score-table .table-wrap,
#raw-score-table .wrap,
#raw-score-table .wrap-inner {{
    max-height: {dataframe_height(RAW_SCORE_DF)}px;
    overflow-y: auto;
}}

#unweighted-z-table .table-wrap,
#unweighted-z-table .wrap,
#unweighted-z-table .wrap-inner {{
    max-height: {dataframe_height(UNWEIGHTED_Z_DF)}px;
    overflow-y: auto;
}}

#weighted-z-table .table-wrap,
#weighted-z-table .wrap,
#weighted-z-table .wrap-inner {{
    max-height: {dataframe_height(WEIGHTED_Z_DF)}px;
    overflow-y: auto;
}}
"""

demo = gr.Blocks(css=custom_css + DATAFRAME_HEIGHT_CSS)


with demo:
    gr.HTML(custom_js)
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("Result", elem_id="result-tab", id=0):
            with gr.Tabs(elem_classes="tab-buttons") as nested_tabs:
                with gr.TabItem("Raw Score"):
                    create_gr_dataframe(
                        RAW_SCORE_DF,
                        wrap=True,
                        column_widths=RAW_SCORE_COLUMN_WIDTHS,
                        row_count=(len(RAW_SCORE_DF), "fixed"),
                        height=dataframe_height(RAW_SCORE_DF),
                        elem_id="raw-score-table",
                    )
                with gr.TabItem("Weights"):
                    gr.HTML(create_weights_table_html(), elem_id="weights-table")
                with gr.TabItem("Unweighted Z-score"):
                    create_gr_dataframe(
                        UNWEIGHTED_Z_DF,
                        wrap=True,
                        column_widths=UNWEIGHTED_Z_COLUMN_WIDTHS,
                        row_count=(len(UNWEIGHTED_Z_DF), "fixed"),
                        height=dataframe_height(UNWEIGHTED_Z_DF),
                        elem_id="unweighted-z-table",
                    )
                with gr.TabItem("Weighted Z-score"):
                    create_gr_dataframe(
                        WEIGHTED_Z_DF,
                        wrap=True,
                        column_widths=WEIGHTED_Z_COLUMN_WIDTHS,
                        row_count=(len(WEIGHTED_Z_DF), "fixed"),
                        height=dataframe_height(WEIGHTED_Z_DF),
                        elem_id="weighted-z-table",
                    )

        with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2):
            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")


scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=1800)
scheduler.start()
demo.queue(default_concurrency_limit=40).launch()