new-model-mode (#4)

- adding new model mode (0ad4e9bc017167a8624cc89852342440e7adef19)

.gitignore

@@ -176,4 +176,8 @@ cython_debug/
 evaluation_results/
 images/
 hf_cache/
-*.lock
+*.lock
+
+# macOS
+.DS_Store
+

app.py

@@ -2349,8 +2349,8 @@ qwen_combined_dir = _resolve_data_path(
     space_root.parent / "qwen_image_bench_combined",
 )
 qwen_path = _resolve_data_path(
-    data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
-    space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
+    data_dir / "qwen_image_bench_model_price_and_median_generation_time_10_august.csv",
+    space_root.parent / "qwen_image_bench_model_price_and_median_generation_time_10_august.csv",
 )
 aa_path = _resolve_data_path(
     data_dir / "artificial_analysis_text_to_image_leaderboard.csv",

data/qwen_image_bench_model_price_and_median_generation_time.csv

@@ -1,60 +0,0 @@
-Model,Price / Image (USD),Median Generation Time (s),Min Generation Time (s),P-Judge Overall,Raw Win Rate,Rapidata Elo,Datapoint Elo,Benchmark.ai Elo
-reve_2_1,N/A,N/A,28.3,,,,,1173.2
-ideogram_4_0_quality,N/A,N/A,66.6,,,,,1131.0
-gpt_image_2,0.21,77.8,77.8,59.02083099999998,,1172.58,1116,1124.5
-nano_banana_2_0,N/A,N/A,N/A,56.362956,59.4%,1071.79,1067,1056.3
-gpt_image_1_5,0.135,38.0,38.0,57.864434999999986,64.5%,1102.09,1064,910.3
-hidream_i1_dev,0.0086,2.823569217998738,2.06,49.17682099999999,51.7%,999.17,984,
-gpt_image_1,0.167,38.8,38.8,54.975741000000006,,1095.39,987,
-imagen_4_0_ultra,0.06,11.4,11.4,53.385603999999965,59.7%,1075.22,1023,
-flux_2_flex,0.06,10.856533817990567,8.07,53.9245676767677,57.8%,1054.34,1018,921.3
-qwen_image,0.025,4.8,4.8,51.561746,51.5%,1073.72,1005,975.6
-seedream_5_0,N/A,N/A,N/A,55.715153,,1070.35,1012,
-nano_banana_pro,0.134,17.2,17.2,56.452189898989914,58.0%,1029.51,1045,1102.4
-hidream_i1_fast,0.0051,9.920469530501578,1.40,48.89298600000002,50.1%,1024.67,984,
-imagen_4_fast,0.02,3.7531301500021073,2.71,50.155055208333344,,972.6,981,
-seedream_4_5,0.04,16.6,16.6,55.66328800000001,,1048.96,1035,964.2
-seedream_4_0,0.03,12.2,12.2,55.241443999999994,,1050.41,1035,
-p_image_2_ideogram_low_1k,0.0075,2.59,1.55,54.827397,53.2%,1000.07,1009,1103.9
-#p_image_2_ideogram_low_2k,0.016,5.17,4.25,53.971723,48.4%,1074.91,1003,
-qwen_image_2_0_pro,0.035,35.5,35.5,56.181776,,1033.25,1017,959.9
-juggernaut_base_flux,0.035,4.115834823496698,3.84,49.474676,46.4%,1038.22,972,
-flux_2_pro,N/A,N/A,N/A,54.43423900000002,,993.54,1019,1011.6
-z_image,0.005,1.5122045120006078,1.22,49.946227999999984,48.1%,1028.1,1001,
-p_image_2_ideogram_high_1k,0.015,4.28,3.07,55.754507999999994,52.1%,972.92,1022,1104.0
-#p_image_2_ideogram_high_2k,0.03,8.69,7.12,54.757842000000004,49.4%,1074.91,1007,
-qwen_image_2512,0.02,19.1,19.1,51.677326,,1029.6,1009,
-flux_2_max,0.07,26.4,26.4,54.047976,60.9%,955.19,1028,1001.6
-flux_1_1_pro_ultra,0.06,9.026992494000297,6.20,50.358445,52.0%,979.56,995,
-juggernaut_pro_flux,0.055,3.699636150500737,3.36,49.741183,46.4%,972.14,964,
-flux_dev,0.025,1.6931055715031107,1.49,48.241479,42.1%,925.04,940,
-p_image_2_ideogram_very_low_1k,0.003,2.56,1.49,53.51894200000001,48.1%,959.86,995,1071.7
-#p_image_2_ideogram_very_low_2k,0.006,3.94,3.05,53.68248699999998,47.2%,962.38,980,
-#p_image_2_ideogram_very_low_1k_no_upsampling,0.005,0.82,,46.33,,,,1071.7
-#p_image_2_ideogram_very_low_2k_no_upsampling,0.005,2.18,,46.39,,,,1071.7
-#p_image_2_ideogram_low_1k_no_upsampling,0.01,3.33,,45.32,,,,1103.9
-#p_image_2_ideogram_low_2k_no_upsampling,0.01,3.34,,46.25,,,,1103.9
-#p_image_2_ideogram_medium_1k_no_upsampling,0.015,2.13,,46.88,,,,1115.1
-#p_image_2_ideogram_medium_2k_no_upsampling,0.015,5.55,,47.11,,,,1115.1
-#p_image_2_ideogram_high_1k_no_upsampling,0.03,3.11,,46.88,,,,1104.0
-#p_image_2_ideogram_high_2k_no_upsampling,0.03,5.55,,47.06,,,,1104.0
-hidream_i1_full,0.014,6.008430051002506,5.69,46.82559300000002,36.9%,955.46,944,
-flux_2_dev,0.025,4.310278721997747,4.03,52.71644489795918,53.1%,1007.6,1021,942.0
-imagen_4_0,0.04,14.1,14.1,52.08996199999999,53.5%,979.62,1005,
-wan_2_2_image,0.02,3.005390542501118,2.96,48.19959399999999,,944.87,960,
-flux_krea,0.025,1.7150160090022837,1.7150160090022837,50.35734,50.0%,919.73,975,
-p_image_2_ideogram_medium_1k,0.01,3.06,2.05,54.719193000000004,51.7%,941.46,1002,1115.1
-#p_image_2_ideogram_medium_2k,0.02,7.44,6.47,54.23124444444446,48.1%,949.35,1000,
-glm_image,0.05,188.2,188.2,51.42623399999999,,923.35,953,
-p_image,0.005,1.0640762715011078,0.95,48.75217099999999,44.8%,924.37,961,1098.7
-hunyuanimage_3_0,0.09,41.0,41.0,52.32440099999998,52.4%,1009.61,979,765.3
-juggernaut_lightning_flux,0.006,1.1787893719956628,0.93,48.30471699999998,40.5%,916.68,929,
-flux_1_1_pro,0.04,3.0104645500032348,2.34,49.92882700000001,50.6%,925.04,984,
-flux_schnell,0.003,0.8411653029907029,0.80,46.685981818181816,34.8%,892.32,915,
-kling_v2_1,N/A,N/A,N/A,51.044512,,870.04,981,
-#p_image_2_ideogram_very_high_high_1k,0.075,9.34,7.25,58.45,,,1025,
-#p_image_2_ideogram_very_high_low_1k,0.0375,26.17,10.44,57.59,,,1024,
-#p_image_2_ideogram_very_high_medium_1k,0.05,9.28,5.96,57.88,,,1020,
-#p_image_2_ideogram_very_high_very_low_1k,0.015,26.52,10.60,56.92,,,1002,
-#p_image_2_ideogram_final_1k,0.0375,11.17,5.55,58.28,,,,
-#p_image_2_ideogram_final_2k,0.075,14.34,9.96,57.68,,,,

data/qwen_image_bench_model_price_and_median_generation_time_10_august.csv

@@ -0,0 +1,64 @@
+Rapidata Model,Price / Image (USD),Median Generation Time (s),Min Generation Time (s),P-Judge Overall,Raw Win Rate,Rapidata Elo,Datapoint Elo,Benchmark.ai Elo
+reve_2_1,N/A,N/A,28.3,,,,,1173.2
+ideogram_4_0_quality,N/A,N/A,66.6,,,,,1131.0
+gpt_image_2,0.21,77.8,77.8,59.02083099999998,65.6%,1172.58,1110,1124.5
+nano_banana_2_0,N/A,N/A,N/A,56.362956,59.2%,1071.79,1063,1056.3
+gpt_image_1_5,0.135,38.0,38.0,57.864434999999986,58.9%,1102.09,1060,910.3
+hidream_i1_dev,0.0086,2.823569217998738,2.06,49.17682099999999,47.7%,999.17,983,
+gpt_image_1,0.167,38.8,38.8,54.975741000000006,47.4%,1095.39,984,
+imagen_4_0_ultra,0.06,11.4,11.4,53.385603999999965,52.9%,1075.22,1019,
+flux_2_flex,0.06,10.856533817990567,8.07,53.9245676767677,52.4%,1054.34,1014,921.3
+#flux_2_turbo,0.008,2.14,1.81,53.02,,,1003,
+#flux_2_flash,0.005,1.43,1.03,52.03,,,1001,
+qwen_image,0.025,4.8,4.8,51.561746,50.3%,1073.72,1002,975.6
+seedream_5_0,N/A,N/A,N/A,55.715153,51.6%,1070.35,1010,
+nano_banana_pro,0.134,17.2,17.2,56.452189898989914,56.4%,1029.51,1041,1102.4
+hidream_i1_fast,0.0051,9.920469530501578,1.40,48.89298600000002,47.2%,1024.67,980,
+imagen_4_fast,0.02,3.7531301500021073,2.71,50.155055208333344,47.2%,972.6,980,
+seedream_4_5,0.04,16.6,16.6,55.66328800000001,54.7%,1048.96,1032,964.2
+seedream_4_0,0.03,12.2,12.2,55.241443999999994,54.9%,1050.41,1032,
+p_image_2_ideogram_low_1k,0.0075,2.59,1.55,54.827397,50.8%,1000.07,1006,1103.9
+#p_image_2_ideogram_low_2k,0.016,5.17,4.25,53.971723,50.0%,1074.91,1000,1103.9
+qwen_image_2_0_pro,0.035,35.5,35.5,56.181776,52.2%,1033.25,1014,959.9
+juggernaut_base_flux,0.035,4.115834823496698,3.84,49.474676,46.1%,1038.22,971,
+flux_2_pro,N/A,N/A,N/A,54.43423900000002,52.2%,993.54,1014,1011.6
+z_image,0.005,1.5122045120006078,1.22,49.946227999999984,49.9%,1028.1,999,
+p_image_2_ideogram_high_1k,0.015,4.28,3.07,55.754507999999994,52.5%,972.92,1017,1104.0
+#p_image_2_ideogram_high_2k,0.06,8.69,7.12,54.757842000000004,50.7%,1074.91,1006,1104.0
+qwen_image_2512,0.02,19.1,19.1,51.677326,50.9%,1029.6,1005,
+flux_2_max,0.07,26.4,26.4,54.047976,53.9%,955.19,1026,1001.6
+flux_1_1_pro_ultra,0.06,9.026992494000297,6.20,50.358445,48.6%,979.56,990,
+juggernaut_pro_flux,0.055,3.699636150500737,3.36,49.741183,44.7%,972.14,963,
+flux_dev,0.025,1.6931055715031107,1.49,48.241479,41.5%,925.04,941,
+p_image_2_ideogram_very_low_1k,0.003,2.56,1.49,53.51894200000001,49.1%,959.86,994,1071.7
+#p_image_2_ideogram_very_low_2k,0.006,3.94,3.05,53.68248699999998,46.3%,962.38,976,1071.7
+#p_image_2_ideogram_very_low_1k_no_upsampling,0.005,0.82,,46.33,,,,1071.7
+#p_image_2_ideogram_very_low_2k_no_upsampling,0.005,2.18,,46.39,,,,1071.7
+#p_image_2_ideogram_low_1k_no_upsampling,0.01,3.33,,45.32,,,,1103.9
+#p_image_2_ideogram_low_2k_no_upsampling,0.01,3.34,,46.25,,,,1103.9
+#p_image_2_ideogram_medium_1k_no_upsampling,0.015,2.13,,46.88,,,,1115.1
+#p_image_2_ideogram_medium_2k_no_upsampling,0.015,5.55,,47.11,,,,1115.1
+#p_image_2_ideogram_high_1k_no_upsampling,0.03,3.11,,46.88,,,,1104.0
+#p_image_2_ideogram_high_2k_no_upsampling,0.03,5.55,,47.06,,,,1104.0
+hidream_i1_full,0.014,6.008430051002506,5.69,46.82559300000002,41.4%,955.46,941,
+flux_2_dev,0.025,4.310278721997747,4.03,52.71644489795918,52.5%,1007.6,1016,942.0
+imagen_4_0,0.04,14.1,14.1,52.08996199999999,50.4%,979.62,1002,
+wan_2_2_image,0.02,3.005390542501118,2.96,48.19959399999999,44.1%,944.87,958,
+flux_krea,0.025,1.7150160090022837,1.7150160090022837,50.35734,46.2%,919.73,973,
+p_image_2_ideogram_medium_1k,0.01,3.06,2.05,54.719193000000004,49.8%,941.46,999,1115.1
+#p_image_2_ideogram_medium_2k,0.02,7.44,6.47,54.23124444444446,49.3%,949.35,996,1115.1
+glm_image,0.05,188.2,188.2,51.42623399999999,42.9%,923.35,950,
+p_image,0.005,1.0640762715011078,0.95,48.75217099999999,44.3%,924.37,960,1098.7
+hunyuanimage_3_0,0.09,41.0,41.0,52.32440099999998,46.6%,1009.61,977,765.3
+juggernaut_lightning_flux,0.006,1.1787893719956628,0.93,48.30471699999998,40.0%,916.68,930,
+flux_1_1_pro,0.04,3.0104645500032348,2.34,49.92882700000001,47.7%,925.04,984,
+flux_schnell,0.003,0.8411653029907029,0.80,46.685981818181816,37.6%,892.32,914,
+kling_v2_1,N/A,N/A,N/A,51.044512,47.2%,870.04,980,
+#p_image_2_ideogram_very_high_high_1k,0.075,9.34,7.25,58.45,52.9%,,1024,
+#p_image_2_ideogram_very_high_low_1k,0.0375,26.17,10.44,57.59,52.9%,,1023,
+#p_image_2_ideogram_very_high_medium_1k,0.05,9.28,5.96,57.88,52.3%,,1019,
+#p_image_2_ideogram_very_high_very_low_1k,0.015,26.52,10.60,56.92,49.4%,,1001,
+p_image_2_ideogram_very_high_1k,0.033,11.17,5.55,58.28,55.5%,,1034,
+#p_image_2_ideogram_very_high_2k,0.066,14.34,9.96,57.68,53.4%,,1020,
+#p_image_2_ideogram_very_high_john_1k,,8.53,6.43,58.19,,,1015,
+#p_image_2_ideogram_very_high_john_2k,,13.86,10.64,57.75,,,1002,

model_display.py

@@ -87,6 +87,7 @@ MODEL_DISPLAY_NAMES = {
     "p_image_2_ideogram_high_1k": "P-Image-Ideogram High 1K",
     "p_image_2_ideogram_high_2k": "P-Image-Ideogram High 2K",
     "P-Image-Ideogram (High)": "P-Image-Ideogram High",
+    "p_image_2_ideogram_very_high_1k": "P-Image-Ideogram Very High 1K",
     # Others overlapping P-Bench
     "z_image": "Z-Image",
     "glm_image": "GLM-Image",

ui.py

@@ -251,6 +251,14 @@ def _dataset_has_samples(datasets, dataset_id):
     return bool(dataset and dataset.get("samples"))
 
 
+def _sample_model_ids(datasets, dataset_id):
+    dataset = _item(datasets, dataset_id)
+    samples = dataset.get("samples") if dataset else None
+    if not samples:
+        return set()
+    return set(samples.get("models") or [])
+
+
 def _dataset_has_pareto(datasets, dataset_id):
     dataset = _item(datasets, dataset_id)
     columns = getattr(dataset.get("data") if dataset else None, "columns", [])
@@ -321,23 +329,25 @@ def _metric_dropdown_value(metric_id):
     ]
 
 
-def _model_choices(datasets, dataset_id):
+def _model_choices(datasets, dataset_id, *, require_samples=False):
     cached = _MODEL_CHOICES_CACHE.get(dataset_id)
-    if cached is not None:
+    if cached is None:
+        dataset = _item(datasets, dataset_id)
+        data = dataset.get("data") if dataset else None
+        if data is None or "Model" not in getattr(data, "columns", []):
+            cached = []
+        else:
+            models = data["Model"].dropna().astype(str).unique().tolist()
+            # (label, value) so the UI shows the shared name but filters on the raw id.
+            cached = sorted(
+                ((display_model_name(model), model) for model in models),
+                key=lambda item: item[0].casefold(),
+            )
+        _MODEL_CHOICES_CACHE[dataset_id] = cached
+    if not require_samples:
         return cached
-    dataset = _item(datasets, dataset_id)
-    data = dataset.get("data") if dataset else None
-    if data is None or "Model" not in getattr(data, "columns", []):
-        _MODEL_CHOICES_CACHE[dataset_id] = []
-        return []
-    models = data["Model"].dropna().astype(str).unique().tolist()
-    # (label, value) so the UI shows the shared name but filters on the raw id.
-    choices = sorted(
-        ((display_model_name(model), model) for model in models),
-        key=lambda item: item[0].casefold(),
-    )
-    _MODEL_CHOICES_CACHE[dataset_id] = choices
-    return choices
+    allowed = _sample_model_ids(datasets, dataset_id)
+    return [choice for choice in cached if choice[1] in allowed]
 
 
 def _model_choice_values(choices):
@@ -1163,10 +1173,10 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
             gr.Markdown(
                 "<p class='filter-help'>"
                 "These filters apply to Leaderboards, Pareto plots, and Samples. "
-                "On Samples, only datasets we have generations for are listed. "
-                "On Pareto plots, only datasets with price or generation time "
-                "are listed. Search in Models, or leave it empty to include "
-                "every model."
+                "On Samples, only datasets and models we have generations for "
+                "are listed. On Pareto plots, only datasets with price or "
+                "generation time are listed. Search in Models, or leave it "
+                "empty to include every model."
                 "</p>",
                 elem_classes="filter-help-host",
             )
@@ -1369,12 +1379,16 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
             with gr.TabItem("About", id=TAB_ABOUT) as about_tab:
                 render_about()
 
-    def _synced_filters(dataset_id, metric_id, models, *, clear_metric=False):
+    def _synced_filters(
+        dataset_id, metric_id, models, *, clear_metric=False, require_samples=False
+    ):
         if clear_metric:
             metric_id = []
         else:
             metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
-        model_choices = _model_choices(datasets, dataset_id)
+        model_choices = _model_choices(
+            datasets, dataset_id, require_samples=require_samples
+        )
         model_values = set(_model_choice_values(model_choices))
         models = [model for model in (models or []) if model in model_values]
         metric_choices = _metric_dropdown_choices(datasets, metrics, dataset_id)
@@ -1588,10 +1602,20 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
         dataset_changed = source == "dataset" and dataset_id != view_state.get(
             "dataset_id"
         )
-
+        can_pareto = _dataset_has_pareto(datasets, dataset_id)
+        can_samples = _dataset_has_samples(datasets, dataset_id)
+        selected_tab = tab
         if source == "dataset":
+            if tab == TAB_SAMPLES and not can_samples:
+                selected_tab = TAB_LEADERBOARDS
+            elif tab == TAB_PARETO and not can_pareto:
+                selected_tab = TAB_LEADERBOARDS
             synced = _synced_filters(
-                dataset_id, metric_id, models, clear_metric=dataset_changed
+                dataset_id,
+                metric_id,
+                models,
+                clear_metric=dataset_changed,
+                require_samples=selected_tab == TAB_SAMPLES,
             )
             dataset_id, metric_id, models = synced[:3]
             metric_update, models_update = synced[3], synced[4]
@@ -1629,20 +1653,13 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
         ):
             return None
 
-        selected_tab = tab
         extras = (
             list(platform_value or []),
             list(owner_value or []),
             list(optimized_value or []),
         )
         extra_updates = None
-        can_pareto = _dataset_has_pareto(datasets, dataset_id)
-        can_samples = _dataset_has_samples(datasets, dataset_id)
         if source == "dataset":
-            if tab == TAB_SAMPLES and not can_samples:
-                selected_tab = TAB_LEADERBOARDS
-            elif tab == TAB_PARETO and not can_pareto:
-                selected_tab = TAB_LEADERBOARDS
             view = resolve_view(datasets, metrics, dataset_id, metric_id)
             extra_updates = _leaderboard_extras(
                 view["data"] if view else None,
@@ -1822,7 +1839,17 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
             )
         dataset_update = _dataset_dropdown_update(datasets, tab, dataset_id)
         metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
-        models = list(models or [])
+        require_samples = tab == TAB_SAMPLES
+        model_choices = _model_choices(
+            datasets, dataset_id, require_samples=require_samples
+        )
+        allowed_models = set(_model_choice_values(model_choices))
+        models = [model for model in (models or []) if model in allowed_models]
+        models_update = (
+            gr.update(choices=model_choices, value=models)
+            if (prev_tab == TAB_SAMPLES) != require_samples
+            else gr.skip()
+        )
         view_state["current_tab"] = tab
         view_state["dataset_id"] = dataset_id
         view_state["metric_id"] = metric_id
@@ -1859,7 +1886,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
             filters_vis,
             dataset_update,
             metric_vis,
-            gr.skip(),
+            models_update,
             *lb_filters,
         )
         tab_select = (