.gitignore CHANGED
@@ -176,8 +176,4 @@ cython_debug/
176
  evaluation_results/
177
  images/
178
  hf_cache/
179
- *.lock
180
-
181
- # macOS
182
- .DS_Store
183
-
 
176
  evaluation_results/
177
  images/
178
  hf_cache/
179
+ *.lock
 
 
 
 
README.md CHANGED
@@ -30,6 +30,8 @@ pip install "gradio==5.19.0" pandas -r requirements.txt
30
  python app.py
31
  ```
32
 
 
 
33
  `requirements.txt` lists Plotly. Gradio and pandas are required locally;
34
  Hugging Face Spaces installs Gradio from the YAML `sdk_version` above.
35
 
 
30
  python app.py
31
  ```
32
 
33
+ The app is served at `http://127.0.0.1:7860`.
34
+
35
  `requirements.txt` lists Plotly. Gradio and pandas are required locally;
36
  Hugging Face Spaces installs Gradio from the YAML `sdk_version` above.
37
 
app.py CHANGED
@@ -62,9 +62,6 @@ custom_css = """
62
  --pruna-accordion-bg: rgba(255, 255, 255, 0.02);
63
  --pruna-accordion-border: rgba(216, 180, 254, 0.15);
64
  --pruna-dropdown-hover: #2a1844;
65
- --pruna-toggle-track: var(--pruna-bg-header);
66
- --pruna-toggle-thumb: var(--pruna-bg-elevated);
67
- --pruna-toggle-thumb-shadow: 0 1px 2px rgba(0, 0, 0, 0.45), inset 0 1px rgba(255, 255, 255, 0.06);
68
  color-scheme: dark;
69
  }
70
 
@@ -108,27 +105,13 @@ custom_css = """
108
  --pruna-accordion-bg: var(--pruna-bg-card);
109
  --pruna-accordion-border: var(--pruna-border);
110
  --pruna-dropdown-hover: #f3e8ff;
111
- --pruna-toggle-track: var(--pruna-bg-header);
112
- --pruna-toggle-thumb: var(--pruna-bg-card);
113
- --pruna-toggle-thumb-shadow: 0 1px 2px rgba(88, 28, 135, 0.12);
114
  color-scheme: light;
115
  }
116
 
117
- html {
118
  width: 100% !important;
119
  max-width: 100% !important;
120
- min-width: 0 !important;
121
- overflow-x: hidden;
122
- overflow-y: auto;
123
- -webkit-text-size-adjust: 100%;
124
- text-size-adjust: 100%;
125
- -webkit-tap-highlight-color: transparent;
126
- }
127
- body, gradio-app {
128
- width: 100% !important;
129
- max-width: 100% !important;
130
- min-width: 0 !important;
131
- overflow: visible;
132
  -webkit-tap-highlight-color: transparent;
133
  }
134
  html, body, .gradio-container, .main {
@@ -151,25 +134,18 @@ button, a, label, input, select, textarea,
151
  /* Subtle depth — not a marketing-site hero glow */
152
  body, .gradio-container {
153
  background-image: var(--pruna-glow) !important;
154
- background-repeat: no-repeat !important;
155
- background-attachment: scroll !important;
156
- }
157
- @media (min-width: 701px) and (hover: hover) and (pointer: fine) {
158
- body, .gradio-container {
159
- background-attachment: fixed !important;
160
- }
161
  }
162
 
163
  .gradio-container {
164
  width: 100% !important;
165
  max-width: 1200px !important;
166
- min-width: 0 !important;
167
  margin: 0 auto !important;
168
  padding-top: 0 !important;
169
  padding-left: 20px !important;
170
  padding-right: 20px !important;
171
  box-sizing: border-box !important;
172
- overflow-x: hidden;
173
  }
174
  .gradio-container .main,
175
  .gradio-container .wrap,
@@ -190,7 +166,6 @@ body, .gradio-container {
190
  .workspace-filters,
191
  .view-filters {
192
  max-width: 100% !important;
193
- min-width: 0 !important;
194
  }
195
 
196
  /* —— App header (P-Bench only) —— */
@@ -325,9 +300,6 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
325
  background: transparent !important;
326
  box-shadow: none !important;
327
  }
328
- /* Flatten tabs so the bar sits above shared filters. display:contents is
329
- the fallback; Safari can drop or mis-order those children, so browsers
330
- with subgrid use the grid layout below instead. */
331
  .workspace-shell > .tabs,
332
  .workspace-shell > .main-tabs,
333
  .workspace-shell > .block:not(.workspace-filters),
@@ -351,49 +323,12 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
351
  margin: 0 0 16px !important;
352
  justify-content: center !important;
353
  width: 100% !important;
354
- overflow: visible !important;
355
- }
356
- @supports (grid-template-rows: subgrid) {
357
- .workspace-shell,
358
- .workspace-shell.block,
359
- .workspace-shell.column,
360
- .workspace-shell.gap {
361
- display: grid !important;
362
- grid-template-columns: minmax(0, 1fr) !important;
363
- grid-template-rows: auto auto auto !important;
364
- align-content: start !important;
365
- }
366
- .workspace-shell > .tabs,
367
- .workspace-shell > .main-tabs,
368
- .workspace-shell .tabs.main-tabs {
369
- display: grid !important;
370
- grid-template-columns: minmax(0, 1fr) !important;
371
- grid-template-rows: subgrid !important;
372
- grid-column: 1 !important;
373
- grid-row: 1 / 4 !important;
374
- position: static !important;
375
- }
376
- .main-tabs > .tab-wrapper {
377
- grid-row: 1 !important;
378
- order: 0 !important;
379
- }
380
- .workspace-filters {
381
- grid-column: 1 !important;
382
- grid-row: 2 !important;
383
- order: 0 !important;
384
- }
385
- .main-tabs .tabitem {
386
- grid-row: 3 !important;
387
- order: 0 !important;
388
- min-width: 0 !important;
389
- }
390
  }
391
  .main-tabs .tab-container {
392
  height: auto !important;
393
- min-height: 0 !important;
394
  justify-content: center !important;
395
  flex-wrap: wrap !important;
396
- overflow: visible !important;
397
  max-width: 100% !important;
398
  gap: 2px;
399
  }
@@ -510,8 +445,6 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
510
  .app-header-brand h1,
511
  .gradio-container .app-header-brand h1 {
512
  font-size: 1.55rem !important;
513
- width: auto !important;
514
- max-width: 100% !important;
515
  }
516
  .app-header-tagline {
517
  font-size: 0.88rem !important;
@@ -588,29 +521,17 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
588
  left: 0 !important;
589
  width: 2.4rem;
590
  min-width: 2.4rem;
591
- box-shadow: 6px 0 8px -6px rgba(0, 0, 0, 0.45);
592
  }
593
  .ranking-table .model-cell,
594
- .prose .ranking-table .model-cell {
595
- position: static !important;
596
- left: auto !important;
597
- z-index: auto;
598
- min-width: 140px;
599
- max-width: none;
600
- background: transparent !important;
601
- }
602
- .ranking-table th.model-cell,
603
- .prose .ranking-table th.model-cell {
604
  position: sticky !important;
605
- top: 0 !important;
606
- left: auto !important;
607
- z-index: 3;
608
- min-width: 140px;
609
- max-width: none;
610
- background: var(--pruna-bg-header) !important;
611
  }
612
- .ranking-table tbody tr:hover .model-cell {
613
- background: var(--pruna-table-hover) !important;
614
  }
615
  .compare-controls,
616
  .compare-controls.row,
@@ -683,7 +604,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
683
  .view-filters {
684
  display: flex !important;
685
  flex-wrap: wrap !important;
686
- align-items: flex-end !important;
687
  gap: 12px !important;
688
  margin: 0;
689
  overflow: visible !important;
@@ -691,7 +612,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
691
  .view-filters > div,
692
  .view-filters > .block,
693
  .view-filters > .form {
694
- flex: 1 1 0% !important;
695
  min-width: 0 !important;
696
  }
697
  .view-filters > .block,
@@ -843,10 +764,6 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
843
  line-height: 1.45 !important;
844
  font-weight: 400 !important;
845
  }
846
- .view-help + .view-help,
847
- .prose .view-help + .view-help {
848
- margin-top: 0.45rem !important;
849
- }
850
  .view-filters span[data-testid="block-info"],
851
  .view-filters .info,
852
  .view-filters .block-info {
@@ -984,7 +901,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
984
  .leaderboard-controls {
985
  display: flex !important;
986
  flex-wrap: wrap !important;
987
- align-items: flex-end !important;
988
  gap: 10px !important;
989
  margin-bottom: 12px;
990
  overflow: visible !important;
@@ -1400,9 +1317,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
1400
  box-shadow: none !important;
1401
  }
1402
  .compare-controls .compare-prompt-count .head {
1403
- display: block !important;
1404
- grid-column: 1 / -1;
1405
- grid-row: 1;
1406
  margin: 0 !important;
1407
  }
1408
  .compare-controls .compare-prompt-count .head label {
@@ -1576,182 +1491,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
1576
  margin: 0 !important;
1577
  }
1578
  .compare-row { display: grid; gap: 12px; min-width: 0; width: 100%; }
1579
- .compare-prompt-text { overflow-wrap: anywhere; word-break: break-word; }
1580
- .pareto-heading-row,
1581
- .pareto-heading-row.row,
1582
- .pareto-heading-row .form {
1583
- display: flex !important;
1584
- flex-wrap: wrap !important;
1585
- align-items: center !important;
1586
- gap: 8px 12px !important;
1587
- width: 100% !important;
1588
- margin-bottom: 0.4rem !important;
1589
- }
1590
- .pareto-heading-row .pareto-subhead,
1591
- .pareto-heading-row > div:first-child,
1592
- .pareto-heading-row .form > div:first-child {
1593
- flex: 1 1 240px !important;
1594
- min-width: 0 !important;
1595
- margin: 0 !important;
1596
- }
1597
- .pareto-heading-row .pareto-scale-control {
1598
- display: flex !important;
1599
- flex-direction: row !important;
1600
- align-items: center !important;
1601
- justify-content: flex-end !important;
1602
- flex: 0 0 auto !important;
1603
- gap: 0 !important;
1604
- margin-left: auto !important;
1605
- max-width: 168px !important;
1606
- padding: 0 !important;
1607
- }
1608
- .pareto-scale-all-row,
1609
- .pareto-scale-all-row.row,
1610
- .pareto-scale-all-row .form {
1611
- display: flex !important;
1612
- flex-direction: row !important;
1613
- flex-wrap: wrap !important;
1614
- align-items: center !important;
1615
- justify-content: flex-start !important;
1616
- gap: 8px 12px !important;
1617
- width: 100% !important;
1618
- margin: 2px 0 14px !important;
1619
- }
1620
- .pareto-scale-all-row .html-container,
1621
- .pareto-scale-all-row .block {
1622
- border: none !important;
1623
- background: transparent !important;
1624
- box-shadow: none !important;
1625
- padding: 0 !important;
1626
- margin: 0 !important;
1627
- width: auto !important;
1628
- flex: 0 0 auto !important;
1629
- }
1630
- .pareto-scale-all-row .html-container {
1631
- flex: 1 1 auto !important;
1632
- min-width: 0 !important;
1633
- }
1634
- .pareto-scale-all-label {
1635
- color: var(--pruna-text-muted);
1636
- font-size: 0.8rem;
1637
- font-weight: 500;
1638
- white-space: nowrap;
1639
- }
1640
- .pareto-scale-all-row .pareto-scale-toggle {
1641
- margin-left: auto !important;
1642
- }
1643
- .pareto-scale-toggle,
1644
- .pareto-scale-toggle.block {
1645
- min-width: 0 !important;
1646
- width: auto !important;
1647
- border: none !important;
1648
- background: transparent !important;
1649
- box-shadow: none !important;
1650
- padding: 0 !important;
1651
- margin: 0 !important;
1652
- }
1653
- .pareto-scale-toggle .wrap,
1654
- .pareto-scale-toggle .form {
1655
- display: block !important;
1656
- width: auto !important;
1657
- margin: 0 !important;
1658
- padding: 0 !important;
1659
- border: none !important;
1660
- background: transparent !important;
1661
- box-shadow: none !important;
1662
- }
1663
- .pareto-scale-toggle legend {
1664
- display: none !important;
1665
- }
1666
- .pareto-scale-toggle fieldset,
1667
- .pareto-scale-toggle .wrap:has(> label),
1668
- .pareto-scale-toggle .form:has(> label) {
1669
- position: relative !important;
1670
- display: grid !important;
1671
- grid-template-columns: 1fr 1fr !important;
1672
- align-items: stretch !important;
1673
- isolation: isolate;
1674
- box-sizing: border-box !important;
1675
- width: max-content !important;
1676
- min-width: 0 !important;
1677
- padding: 4px !important;
1678
- gap: 4px !important;
1679
- border: 1px solid var(--pruna-input-border) !important;
1680
- border-radius: 10px !important;
1681
- background: var(--pruna-toggle-track) !important;
1682
- box-shadow: none !important;
1683
- }
1684
- .pareto-scale-toggle fieldset::before,
1685
- .pareto-scale-toggle .wrap:has(> label)::before,
1686
- .pareto-scale-toggle .form:has(> label)::before {
1687
- content: none !important;
1688
- }
1689
- .pareto-scale-toggle label {
1690
- position: relative !important;
1691
- z-index: 1 !important;
1692
- display: flex !important;
1693
- flex: 1 1 auto !important;
1694
- align-items: center !important;
1695
- justify-content: center !important;
1696
- gap: 0 !important;
1697
- box-sizing: border-box !important;
1698
- min-width: 58px !important;
1699
- min-height: 26px !important;
1700
- margin: 0 !important;
1701
- padding: 5px 12px !important;
1702
- border: none !important;
1703
- border-radius: 6px !important;
1704
- background: transparent !important;
1705
- box-shadow: none !important;
1706
- color: var(--pruna-text-body) !important;
1707
- font-size: 0.72rem !important;
1708
- font-weight: 600 !important;
1709
- line-height: 1.2 !important;
1710
- letter-spacing: 0.01em;
1711
- white-space: nowrap;
1712
- cursor: pointer !important;
1713
- }
1714
- .pareto-scale-toggle-all label {
1715
- min-width: 68px !important;
1716
- min-height: 30px !important;
1717
- padding: 6px 14px !important;
1718
- font-size: 0.85rem !important;
1719
- }
1720
- .pareto-scale-toggle label span {
1721
- margin: 0 !important;
1722
- padding: 0 !important;
1723
- color: inherit !important;
1724
- opacity: 1 !important;
1725
- }
1726
- .pareto-scale-toggle label > * + * {
1727
- margin-left: 0 !important;
1728
- }
1729
- .pareto-scale-toggle label + label::before,
1730
- .pareto-scale-toggle label + label {
1731
- content: none !important;
1732
- border-left: none !important;
1733
- }
1734
- .pareto-scale-toggle input[type="radio"] {
1735
- position: absolute !important;
1736
- appearance: none !important;
1737
- opacity: 0 !important;
1738
- width: 0 !important;
1739
- height: 0 !important;
1740
- margin: 0 !important;
1741
- pointer-events: none !important;
1742
- }
1743
- .pareto-scale-toggle label:hover {
1744
- background: transparent !important;
1745
- color: var(--pruna-text-primary) !important;
1746
- }
1747
- .pareto-scale-toggle label.selected,
1748
- .pareto-scale-toggle label:has(input:checked) {
1749
- background: var(--pruna-toggle-thumb) !important;
1750
- color: var(--pruna-lavender) !important;
1751
- font-weight: 700 !important;
1752
- border-color: transparent !important;
1753
- box-shadow: var(--pruna-toggle-thumb-shadow) !important;
1754
- }
1755
  .pareto-layout,
1756
  .pareto-layout.row,
1757
  .pareto-layout .form {
@@ -1809,7 +1549,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
1809
  .app-header .app-header-brand h1 {
1810
  display: block !important;
1811
  width: max-content !important;
1812
- max-width: 100% !important;
1813
  flex: 0 0 auto !important;
1814
  margin: 0 !important;
1815
  padding: 0 !important;
@@ -2286,7 +2026,7 @@ def load_qwen_combined_dataframe(path):
2286
  df = df[~df["Model"].astype(str).str.startswith("#")].copy()
2287
  df["Model"] = df["Model"].astype(str).str.strip()
2288
 
2289
- df = _as_numeric(
2290
  df,
2291
  [
2292
  "Price / Image (USD)",
@@ -2296,9 +2036,7 @@ def load_qwen_combined_dataframe(path):
2296
  "Rapidata Elo",
2297
  "Datapoint Elo",
2298
  ],
2299
- )
2300
- df = df.drop(columns=["Raw Win Rate"], errors="ignore")
2301
- return df.reset_index(drop=True)
2302
 
2303
 
2304
  df = load_oneig_dataframe(oneig_path)
@@ -2349,8 +2087,8 @@ qwen_combined_dir = _resolve_data_path(
2349
  space_root.parent / "qwen_image_bench_combined",
2350
  )
2351
  qwen_path = _resolve_data_path(
2352
- data_dir / "qwen_image_bench_model_price_and_median_generation_time_10_august.csv",
2353
- space_root.parent / "qwen_image_bench_model_price_and_median_generation_time_10_august.csv",
2354
  )
2355
  aa_path = _resolve_data_path(
2356
  data_dir / "artificial_analysis_text_to_image_leaderboard.csv",
@@ -2371,6 +2109,7 @@ qwen_display_columns = [
2371
  "Datapoint Elo",
2372
  "Rapidata Elo",
2373
  "P-Judge Overall",
 
2374
  "Median Generation Time (s)",
2375
  "Min Generation Time (s)",
2376
  "Price / Image (USD)",
@@ -2578,23 +2317,11 @@ custom_head = """
2578
  return found;
2579
  };
2580
 
2581
- const applyPlotTheme = (gd, mode) => {
2582
- const layout = PLOT_LAYOUT[mode];
2583
- if (!layout || typeof Plotly === "undefined" || !gd) return;
2584
- if (gd.layout && gd.layout.paper_bgcolor === layout.paper_bgcolor) return;
2585
- try { Plotly.relayout(gd, layout); } catch (e) {}
2586
- };
2587
-
2588
- const watchPlotTheme = (gd) => {
2589
- if (!gd || gd.__inferbenchThemeBound) return;
2590
- gd.__inferbenchThemeBound = true;
2591
- gd.addEventListener("plotly_afterplot", () => applyPlotTheme(gd, currentMode()));
2592
- };
2593
-
2594
  const restylePlots = (mode) => {
 
 
2595
  queryAll(".js-plotly-plot").forEach((gd) => {
2596
- watchPlotTheme(gd);
2597
- applyPlotTheme(gd, mode);
2598
  });
2599
  };
2600
 
 
62
  --pruna-accordion-bg: rgba(255, 255, 255, 0.02);
63
  --pruna-accordion-border: rgba(216, 180, 254, 0.15);
64
  --pruna-dropdown-hover: #2a1844;
 
 
 
65
  color-scheme: dark;
66
  }
67
 
 
105
  --pruna-accordion-bg: var(--pruna-bg-card);
106
  --pruna-accordion-border: var(--pruna-border);
107
  --pruna-dropdown-hover: #f3e8ff;
 
 
 
108
  color-scheme: light;
109
  }
110
 
111
+ html, body {
112
  width: 100% !important;
113
  max-width: 100% !important;
114
+ overflow-x: clip;
 
 
 
 
 
 
 
 
 
 
 
115
  -webkit-tap-highlight-color: transparent;
116
  }
117
  html, body, .gradio-container, .main {
 
134
  /* Subtle depth — not a marketing-site hero glow */
135
  body, .gradio-container {
136
  background-image: var(--pruna-glow) !important;
137
+ background-attachment: fixed !important;
 
 
 
 
 
 
138
  }
139
 
140
  .gradio-container {
141
  width: 100% !important;
142
  max-width: 1200px !important;
 
143
  margin: 0 auto !important;
144
  padding-top: 0 !important;
145
  padding-left: 20px !important;
146
  padding-right: 20px !important;
147
  box-sizing: border-box !important;
148
+ overflow-x: clip;
149
  }
150
  .gradio-container .main,
151
  .gradio-container .wrap,
 
166
  .workspace-filters,
167
  .view-filters {
168
  max-width: 100% !important;
 
169
  }
170
 
171
  /* —— App header (P-Bench only) —— */
 
300
  background: transparent !important;
301
  box-shadow: none !important;
302
  }
 
 
 
303
  .workspace-shell > .tabs,
304
  .workspace-shell > .main-tabs,
305
  .workspace-shell > .block:not(.workspace-filters),
 
323
  margin: 0 0 16px !important;
324
  justify-content: center !important;
325
  width: 100% !important;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
326
  }
327
  .main-tabs .tab-container {
328
  height: auto !important;
 
329
  justify-content: center !important;
330
  flex-wrap: wrap !important;
331
+ overflow: hidden !important;
332
  max-width: 100% !important;
333
  gap: 2px;
334
  }
 
445
  .app-header-brand h1,
446
  .gradio-container .app-header-brand h1 {
447
  font-size: 1.55rem !important;
 
 
448
  }
449
  .app-header-tagline {
450
  font-size: 0.88rem !important;
 
521
  left: 0 !important;
522
  width: 2.4rem;
523
  min-width: 2.4rem;
 
524
  }
525
  .ranking-table .model-cell,
526
+ .prose .ranking-table .model-cell,
527
+ .ranking-table th.model-cell {
 
 
 
 
 
 
 
 
528
  position: sticky !important;
529
+ left: 2.4rem !important;
530
+ min-width: 108px;
531
+ max-width: 36vw;
 
 
 
532
  }
533
+ .ranking-table .model-cell strong {
534
+ white-space: nowrap;
535
  }
536
  .compare-controls,
537
  .compare-controls.row,
 
604
  .view-filters {
605
  display: flex !important;
606
  flex-wrap: wrap !important;
607
+ align-items: end !important;
608
  gap: 12px !important;
609
  margin: 0;
610
  overflow: visible !important;
 
612
  .view-filters > div,
613
  .view-filters > .block,
614
  .view-filters > .form {
615
+ flex: 1 1 0 !important;
616
  min-width: 0 !important;
617
  }
618
  .view-filters > .block,
 
764
  line-height: 1.45 !important;
765
  font-weight: 400 !important;
766
  }
 
 
 
 
767
  .view-filters span[data-testid="block-info"],
768
  .view-filters .info,
769
  .view-filters .block-info {
 
901
  .leaderboard-controls {
902
  display: flex !important;
903
  flex-wrap: wrap !important;
904
+ align-items: end !important;
905
  gap: 10px !important;
906
  margin-bottom: 12px;
907
  overflow: visible !important;
 
1317
  box-shadow: none !important;
1318
  }
1319
  .compare-controls .compare-prompt-count .head {
1320
+ display: contents;
 
 
1321
  margin: 0 !important;
1322
  }
1323
  .compare-controls .compare-prompt-count .head label {
 
1491
  margin: 0 !important;
1492
  }
1493
  .compare-row { display: grid; gap: 12px; min-width: 0; width: 100%; }
1494
+ .compare-prompt-text { overflow-wrap: anywhere; }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1495
  .pareto-layout,
1496
  .pareto-layout.row,
1497
  .pareto-layout .form {
 
1549
  .app-header .app-header-brand h1 {
1550
  display: block !important;
1551
  width: max-content !important;
1552
+ max-width: none !important;
1553
  flex: 0 0 auto !important;
1554
  margin: 0 !important;
1555
  padding: 0 !important;
 
2026
  df = df[~df["Model"].astype(str).str.startswith("#")].copy()
2027
  df["Model"] = df["Model"].astype(str).str.strip()
2028
 
2029
+ return _as_numeric(
2030
  df,
2031
  [
2032
  "Price / Image (USD)",
 
2036
  "Rapidata Elo",
2037
  "Datapoint Elo",
2038
  ],
2039
+ ).reset_index(drop=True)
 
 
2040
 
2041
 
2042
  df = load_oneig_dataframe(oneig_path)
 
2087
  space_root.parent / "qwen_image_bench_combined",
2088
  )
2089
  qwen_path = _resolve_data_path(
2090
+ data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
2091
+ space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
2092
  )
2093
  aa_path = _resolve_data_path(
2094
  data_dir / "artificial_analysis_text_to_image_leaderboard.csv",
 
2109
  "Datapoint Elo",
2110
  "Rapidata Elo",
2111
  "P-Judge Overall",
2112
+ "Raw Win Rate",
2113
  "Median Generation Time (s)",
2114
  "Min Generation Time (s)",
2115
  "Price / Image (USD)",
 
2317
  return found;
2318
  };
2319
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2320
  const restylePlots = (mode) => {
2321
+ const layout = PLOT_LAYOUT[mode];
2322
+ if (!layout || typeof Plotly === "undefined") return;
2323
  queryAll(".js-plotly-plot").forEach((gd) => {
2324
+ try { Plotly.relayout(gd, layout); } catch (e) {}
 
2325
  });
2326
  };
2327
 
data/qwen_image_bench_model_price_and_median_generation_time.csv ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Model,Price / Image (USD),Median Generation Time (s),Min Generation Time (s),P-Judge Overall,Raw Win Rate,Rapidata Elo,Datapoint Elo,Benchmark.ai Elo
2
+ reve_2_1,N/A,N/A,28.3,,,,,1173.2
3
+ ideogram_4_0_quality,N/A,N/A,66.6,,,,,1131.0
4
+ gpt_image_2,0.21,77.8,77.8,59.02083099999998,,1172.58,1116,1124.5
5
+ nano_banana_2_0,N/A,N/A,N/A,56.362956,59.4%,1071.79,1067,1056.3
6
+ gpt_image_1_5,0.135,38.0,38.0,57.864434999999986,64.5%,1102.09,1064,910.3
7
+ hidream_i1_dev,0.0086,2.823569217998738,2.06,49.17682099999999,51.7%,999.17,984,
8
+ gpt_image_1,0.167,38.8,38.8,54.975741000000006,,1095.39,987,
9
+ imagen_4_0_ultra,0.06,11.4,11.4,53.385603999999965,59.7%,1075.22,1023,
10
+ flux_2_flex,0.06,10.856533817990567,8.07,53.9245676767677,57.8%,1054.34,1018,921.3
11
+ qwen_image,0.025,4.8,4.8,51.561746,51.5%,1073.72,1005,975.6
12
+ seedream_5_0,N/A,N/A,N/A,55.715153,,1070.35,1012,
13
+ nano_banana_pro,0.134,17.2,17.2,56.452189898989914,58.0%,1029.51,1045,1102.4
14
+ hidream_i1_fast,0.0051,9.920469530501578,1.40,48.89298600000002,50.1%,1024.67,984,
15
+ imagen_4_fast,0.02,3.7531301500021073,2.71,50.155055208333344,,972.6,981,
16
+ seedream_4_5,0.04,16.6,16.6,55.66328800000001,,1048.96,1035,964.2
17
+ seedream_4_0,0.03,12.2,12.2,55.241443999999994,,1050.41,1035,
18
+ p_image_2_ideogram_low_1k,0.0075,2.59,1.55,54.827397,53.2%,1000.07,1009,1103.9
19
+ #p_image_2_ideogram_low_2k,0.016,5.17,4.25,53.971723,48.4%,1074.91,1003,
20
+ qwen_image_2_0_pro,0.035,35.5,35.5,56.181776,,1033.25,1017,959.9
21
+ juggernaut_base_flux,0.035,4.115834823496698,3.84,49.474676,46.4%,1038.22,972,
22
+ flux_2_pro,N/A,N/A,N/A,54.43423900000002,,993.54,1019,1011.6
23
+ z_image,0.005,1.5122045120006078,1.22,49.946227999999984,48.1%,1028.1,1001,
24
+ p_image_2_ideogram_high_1k,0.015,4.28,3.07,55.754507999999994,52.1%,972.92,1022,1104.0
25
+ #p_image_2_ideogram_high_2k,0.03,8.69,7.12,54.757842000000004,49.4%,1074.91,1007,
26
+ qwen_image_2512,0.02,19.1,19.1,51.677326,,1029.6,1009,
27
+ flux_2_max,0.07,26.4,26.4,54.047976,60.9%,955.19,1028,1001.6
28
+ flux_1_1_pro_ultra,0.06,9.026992494000297,6.20,50.358445,52.0%,979.56,995,
29
+ juggernaut_pro_flux,0.055,3.699636150500737,3.36,49.741183,46.4%,972.14,964,
30
+ flux_dev,0.025,1.6931055715031107,1.49,48.241479,42.1%,925.04,940,
31
+ p_image_2_ideogram_very_low_1k,0.003,2.56,1.49,53.51894200000001,48.1%,959.86,995,1071.7
32
+ #p_image_2_ideogram_very_low_2k,0.006,3.94,3.05,53.68248699999998,47.2%,962.38,980,
33
+ #p_image_2_ideogram_very_low_1k_no_upsampling,0.005,0.82,,46.33,,,,1071.7
34
+ #p_image_2_ideogram_very_low_2k_no_upsampling,0.005,2.18,,46.39,,,,1071.7
35
+ #p_image_2_ideogram_low_1k_no_upsampling,0.01,3.33,,45.32,,,,1103.9
36
+ #p_image_2_ideogram_low_2k_no_upsampling,0.01,3.34,,46.25,,,,1103.9
37
+ #p_image_2_ideogram_medium_1k_no_upsampling,0.015,2.13,,46.88,,,,1115.1
38
+ #p_image_2_ideogram_medium_2k_no_upsampling,0.015,5.55,,47.11,,,,1115.1
39
+ #p_image_2_ideogram_high_1k_no_upsampling,0.03,3.11,,46.88,,,,1104.0
40
+ #p_image_2_ideogram_high_2k_no_upsampling,0.03,5.55,,47.06,,,,1104.0
41
+ hidream_i1_full,0.014,6.008430051002506,5.69,46.82559300000002,36.9%,955.46,944,
42
+ flux_2_dev,0.025,4.310278721997747,4.03,52.71644489795918,53.1%,1007.6,1021,942.0
43
+ imagen_4_0,0.04,14.1,14.1,52.08996199999999,53.5%,979.62,1005,
44
+ wan_2_2_image,0.02,3.005390542501118,2.96,48.19959399999999,,944.87,960,
45
+ flux_krea,0.025,1.7150160090022837,1.7150160090022837,50.35734,50.0%,919.73,975,
46
+ p_image_2_ideogram_medium_1k,0.01,3.06,2.05,54.719193000000004,51.7%,941.46,1002,1115.1
47
+ #p_image_2_ideogram_medium_2k,0.02,7.44,6.47,54.23124444444446,48.1%,949.35,1000,
48
+ glm_image,0.05,188.2,188.2,51.42623399999999,,923.35,953,
49
+ p_image,0.005,1.0640762715011078,0.95,48.75217099999999,44.8%,924.37,961,1098.7
50
+ hunyuanimage_3_0,0.09,41.0,41.0,52.32440099999998,52.4%,1009.61,979,765.3
51
+ juggernaut_lightning_flux,0.006,1.1787893719956628,0.93,48.30471699999998,40.5%,916.68,929,
52
+ flux_1_1_pro,0.04,3.0104645500032348,2.34,49.92882700000001,50.6%,925.04,984,
53
+ flux_schnell,0.003,0.8411653029907029,0.80,46.685981818181816,34.8%,892.32,915,
54
+ kling_v2_1,N/A,N/A,N/A,51.044512,,870.04,981,
55
+ #p_image_2_ideogram_very_high_high_1k,0.075,9.34,7.25,58.45,,,1025,
56
+ #p_image_2_ideogram_very_high_low_1k,0.0375,26.17,10.44,57.59,,,1024,
57
+ #p_image_2_ideogram_very_high_medium_1k,0.05,9.28,5.96,57.88,,,1020,
58
+ #p_image_2_ideogram_very_high_very_low_1k,0.015,26.52,10.60,56.92,,,1002,
59
+ #p_image_2_ideogram_final_1k,0.0375,11.17,5.55,58.28,,,,
60
+ #p_image_2_ideogram_final_2k,0.075,14.34,9.96,57.68,,,,
data/qwen_image_bench_model_price_and_median_generation_time_10_august.csv DELETED
@@ -1,64 +0,0 @@
1
- Rapidata Model,Price / Image (USD),Median Generation Time (s),Min Generation Time (s),P-Judge Overall,Raw Win Rate,Rapidata Elo,Datapoint Elo,Benchmark.ai Elo
2
- reve_2_1,N/A,N/A,28.3,,,,,1173.2
3
- ideogram_4_0_quality,N/A,N/A,66.6,,,,,1131.0
4
- gpt_image_2,0.21,77.8,77.8,59.02083099999998,65.6%,1172.58,1110,1124.5
5
- nano_banana_2_0,N/A,N/A,N/A,56.362956,59.2%,1071.79,1063,1056.3
6
- gpt_image_1_5,0.135,38.0,38.0,57.864434999999986,58.9%,1102.09,1060,910.3
7
- hidream_i1_dev,0.0086,2.823569217998738,2.06,49.17682099999999,47.7%,999.17,983,
8
- gpt_image_1,0.167,38.8,38.8,54.975741000000006,47.4%,1095.39,984,
9
- imagen_4_0_ultra,0.06,11.4,11.4,53.385603999999965,52.9%,1075.22,1019,
10
- flux_2_flex,0.06,10.856533817990567,8.07,53.9245676767677,52.4%,1054.34,1014,921.3
11
- #flux_2_turbo,0.008,2.14,1.81,53.02,,,1003,
12
- #flux_2_flash,0.005,1.43,1.03,52.03,,,1001,
13
- qwen_image,0.025,4.8,4.8,51.561746,50.3%,1073.72,1002,975.6
14
- seedream_5_0,N/A,N/A,N/A,55.715153,51.6%,1070.35,1010,
15
- nano_banana_pro,0.134,17.2,17.2,56.452189898989914,56.4%,1029.51,1041,1102.4
16
- hidream_i1_fast,0.0051,9.920469530501578,1.40,48.89298600000002,47.2%,1024.67,980,
17
- imagen_4_fast,0.02,3.7531301500021073,2.71,50.155055208333344,47.2%,972.6,980,
18
- seedream_4_5,0.04,16.6,16.6,55.66328800000001,54.7%,1048.96,1032,964.2
19
- seedream_4_0,0.03,12.2,12.2,55.241443999999994,54.9%,1050.41,1032,
20
- p_image_2_ideogram_low_1k,0.0075,2.59,1.55,54.827397,50.8%,1000.07,1006,1103.9
21
- #p_image_2_ideogram_low_2k,0.016,5.17,4.25,53.971723,50.0%,1074.91,1000,1103.9
22
- qwen_image_2_0_pro,0.035,35.5,35.5,56.181776,52.2%,1033.25,1014,959.9
23
- juggernaut_base_flux,0.035,4.115834823496698,3.84,49.474676,46.1%,1038.22,971,
24
- flux_2_pro,N/A,N/A,N/A,54.43423900000002,52.2%,993.54,1014,1011.6
25
- z_image,0.005,1.5122045120006078,1.22,49.946227999999984,49.9%,1028.1,999,
26
- p_image_2_ideogram_high_1k,0.015,4.28,3.07,55.754507999999994,52.5%,972.92,1017,1104.0
27
- #p_image_2_ideogram_high_2k,0.06,8.69,7.12,54.757842000000004,50.7%,1074.91,1006,1104.0
28
- qwen_image_2512,0.02,19.1,19.1,51.677326,50.9%,1029.6,1005,
29
- flux_2_max,0.07,26.4,26.4,54.047976,53.9%,955.19,1026,1001.6
30
- flux_1_1_pro_ultra,0.06,9.026992494000297,6.20,50.358445,48.6%,979.56,990,
31
- juggernaut_pro_flux,0.055,3.699636150500737,3.36,49.741183,44.7%,972.14,963,
32
- flux_dev,0.025,1.6931055715031107,1.49,48.241479,41.5%,925.04,941,
33
- p_image_2_ideogram_very_low_1k,0.003,2.56,1.49,53.51894200000001,49.1%,959.86,994,1071.7
34
- #p_image_2_ideogram_very_low_2k,0.006,3.94,3.05,53.68248699999998,46.3%,962.38,976,1071.7
35
- #p_image_2_ideogram_very_low_1k_no_upsampling,0.005,0.82,,46.33,,,,1071.7
36
- #p_image_2_ideogram_very_low_2k_no_upsampling,0.005,2.18,,46.39,,,,1071.7
37
- #p_image_2_ideogram_low_1k_no_upsampling,0.01,3.33,,45.32,,,,1103.9
38
- #p_image_2_ideogram_low_2k_no_upsampling,0.01,3.34,,46.25,,,,1103.9
39
- #p_image_2_ideogram_medium_1k_no_upsampling,0.015,2.13,,46.88,,,,1115.1
40
- #p_image_2_ideogram_medium_2k_no_upsampling,0.015,5.55,,47.11,,,,1115.1
41
- #p_image_2_ideogram_high_1k_no_upsampling,0.03,3.11,,46.88,,,,1104.0
42
- #p_image_2_ideogram_high_2k_no_upsampling,0.03,5.55,,47.06,,,,1104.0
43
- hidream_i1_full,0.014,6.008430051002506,5.69,46.82559300000002,41.4%,955.46,941,
44
- flux_2_dev,0.025,4.310278721997747,4.03,52.71644489795918,52.5%,1007.6,1016,942.0
45
- imagen_4_0,0.04,14.1,14.1,52.08996199999999,50.4%,979.62,1002,
46
- wan_2_2_image,0.02,3.005390542501118,2.96,48.19959399999999,44.1%,944.87,958,
47
- flux_krea,0.025,1.7150160090022837,1.7150160090022837,50.35734,46.2%,919.73,973,
48
- p_image_2_ideogram_medium_1k,0.01,3.06,2.05,54.719193000000004,49.8%,941.46,999,1115.1
49
- #p_image_2_ideogram_medium_2k,0.02,7.44,6.47,54.23124444444446,49.3%,949.35,996,1115.1
50
- glm_image,0.05,188.2,188.2,51.42623399999999,42.9%,923.35,950,
51
- p_image,0.005,1.0640762715011078,0.95,48.75217099999999,44.3%,924.37,960,1098.7
52
- hunyuanimage_3_0,0.09,41.0,41.0,52.32440099999998,46.6%,1009.61,977,765.3
53
- juggernaut_lightning_flux,0.006,1.1787893719956628,0.93,48.30471699999998,40.0%,916.68,930,
54
- flux_1_1_pro,0.04,3.0104645500032348,2.34,49.92882700000001,47.7%,925.04,984,
55
- flux_schnell,0.003,0.8411653029907029,0.80,46.685981818181816,37.6%,892.32,914,
56
- kling_v2_1,N/A,N/A,N/A,51.044512,47.2%,870.04,980,
57
- #p_image_2_ideogram_very_high_high_1k,0.075,9.34,7.25,58.45,52.9%,,1024,
58
- #p_image_2_ideogram_very_high_low_1k,0.0375,26.17,10.44,57.59,52.9%,,1023,
59
- #p_image_2_ideogram_very_high_medium_1k,0.05,9.28,5.96,57.88,52.3%,,1019,
60
- #p_image_2_ideogram_very_high_very_low_1k,0.015,26.52,10.60,56.92,49.4%,,1001,
61
- p_image_2_ideogram_very_high_1k,0.033,11.17,5.55,58.28,55.5%,,1034,
62
- #p_image_2_ideogram_very_high_2k,0.066,14.34,9.96,57.68,53.4%,,1020,
63
- #p_image_2_ideogram_very_high_john_1k,,8.53,6.43,58.19,,,1015,
64
- #p_image_2_ideogram_very_high_john_2k,,13.86,10.64,57.75,,,1002,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
model_display.py CHANGED
@@ -87,7 +87,6 @@ MODEL_DISPLAY_NAMES = {
87
  "p_image_2_ideogram_high_1k": "P-Image-Ideogram High 1K",
88
  "p_image_2_ideogram_high_2k": "P-Image-Ideogram High 2K",
89
  "P-Image-Ideogram (High)": "P-Image-Ideogram High",
90
- "p_image_2_ideogram_very_high_1k": "P-Image-Ideogram Very High 1K",
91
  # Others overlapping P-Bench
92
  "z_image": "Z-Image",
93
  "glm_image": "GLM-Image",
 
87
  "p_image_2_ideogram_high_1k": "P-Image-Ideogram High 1K",
88
  "p_image_2_ideogram_high_2k": "P-Image-Ideogram High 2K",
89
  "P-Image-Ideogram (High)": "P-Image-Ideogram High",
 
90
  # Others overlapping P-Bench
91
  "z_image": "Z-Image",
92
  "glm_image": "GLM-Image",
ui.py CHANGED
@@ -1,5 +1,4 @@
1
  from html import escape
2
- from math import ceil, floor, log10
3
  from pathlib import Path
4
  import base64
5
  import random
@@ -26,12 +25,6 @@ MAX_PARETO_METRICS = 8
26
  _PARETO_SLOT_COUNT = 1 + MAX_PARETO_METRICS * 8
27
  _PARETO_PRICE_COLUMN = "Price / Image (USD)"
28
  _PARETO_TIME_COLUMN = "Min Generation Time (s)"
29
- _PARETO_SCALE_CHOICES = [
30
- ("Log", "Logarithmic"),
31
- ("Linear", "Linear"),
32
- ]
33
- _PARETO_SCALE_VALUES = {value for _, value in _PARETO_SCALE_CHOICES}
34
- _PARETO_SCALE_DEFAULT = "Logarithmic"
35
 
36
  TAB_LEADERBOARDS = "leaderboards"
37
  TAB_PARETO = "pareto"
@@ -251,14 +244,6 @@ def _dataset_has_samples(datasets, dataset_id):
251
  return bool(dataset and dataset.get("samples"))
252
 
253
 
254
- def _sample_model_ids(datasets, dataset_id):
255
- dataset = _item(datasets, dataset_id)
256
- samples = dataset.get("samples") if dataset else None
257
- if not samples:
258
- return set()
259
- return set(samples.get("models") or [])
260
-
261
-
262
  def _dataset_has_pareto(datasets, dataset_id):
263
  dataset = _item(datasets, dataset_id)
264
  columns = getattr(dataset.get("data") if dataset else None, "columns", [])
@@ -329,25 +314,23 @@ def _metric_dropdown_value(metric_id):
329
  ]
330
 
331
 
332
- def _model_choices(datasets, dataset_id, *, require_samples=False):
333
  cached = _MODEL_CHOICES_CACHE.get(dataset_id)
334
- if cached is None:
335
- dataset = _item(datasets, dataset_id)
336
- data = dataset.get("data") if dataset else None
337
- if data is None or "Model" not in getattr(data, "columns", []):
338
- cached = []
339
- else:
340
- models = data["Model"].dropna().astype(str).unique().tolist()
341
- # (label, value) so the UI shows the shared name but filters on the raw id.
342
- cached = sorted(
343
- ((display_model_name(model), model) for model in models),
344
- key=lambda item: item[0].casefold(),
345
- )
346
- _MODEL_CHOICES_CACHE[dataset_id] = cached
347
- if not require_samples:
348
  return cached
349
- allowed = _sample_model_ids(datasets, dataset_id)
350
- return [choice for choice in cached if choice[1] in allowed]
 
 
 
 
 
 
 
 
 
 
 
351
 
352
 
353
  def _model_choice_values(choices):
@@ -602,6 +585,7 @@ def _display_label(column):
602
  "Arena Art Elo": "Art",
603
  "Arena Portraits Elo": "Portraits",
604
  "Arena Text Rendering Elo": "Text Rendering",
 
605
  "Median Generation Time (s)": "Median generation time",
606
  "Min Generation Time (s)": "Min generation time",
607
  "Price / Image (USD)": "Price per image",
@@ -686,7 +670,6 @@ def _build_pareto_figure(
686
  x_title,
687
  x_hover_prefix="",
688
  x_hover_suffix="",
689
- x_axis_type="linear",
690
  ):
691
  scatter = (
692
  data[["Model", score_column, x_column]]
@@ -772,29 +755,7 @@ def _build_pareto_figure(
772
  )
773
  axis_font = {"color": "#fafafa", "size": 13}
774
  tick_font = {"color": "#a3a3a3", "size": 12}
775
- x_axis_ticks = {}
776
- if x_axis_type == "log":
777
- positive_x = scatter.loc[scatter[x_column] > 0, x_column].astype(float)
778
- if not positive_x.empty:
779
- minimum = positive_x.min()
780
- maximum = positive_x.max()
781
- tick_values = [
782
- factor * (10**exponent)
783
- for exponent in range(
784
- floor(log10(minimum)),
785
- ceil(log10(maximum)) + 1,
786
- )
787
- for factor in (1, 2, 5)
788
- if minimum * 0.8 <= factor * (10**exponent) <= maximum * 1.2
789
- ]
790
- x_axis_ticks = {
791
- "tickmode": "array",
792
- "tickvals": tick_values,
793
- "ticktext": [f"{value:g}" for value in tick_values],
794
- }
795
  fig.update_xaxes(
796
- type=x_axis_type,
797
- **x_axis_ticks,
798
  showgrid=True,
799
  gridcolor="rgba(74, 57, 98, 0.55)",
800
  zeroline=False,
@@ -813,55 +774,6 @@ def _build_pareto_figure(
813
  return fig
814
 
815
 
816
- def _is_log_scale(scale):
817
- return scale == "Logarithmic"
818
-
819
-
820
- def _pareto_axis_type(scale):
821
- return "log" if _is_log_scale(scale) else "linear"
822
-
823
-
824
- def _pareto_scale_radio(*extra_classes):
825
- return gr.Radio(
826
- choices=_PARETO_SCALE_CHOICES,
827
- value=_PARETO_SCALE_DEFAULT,
828
- show_label=False,
829
- container=False,
830
- elem_classes=["pareto-scale-toggle", *extra_classes],
831
- )
832
-
833
-
834
- def _pareto_plot_heading(title):
835
- with gr.Row(equal_height=False, elem_classes="pareto-heading-row"):
836
- gr.Markdown(f"#### {title}", elem_classes="pareto-subhead")
837
- with gr.Column(min_width=140, elem_classes="pareto-scale-control"):
838
- return _pareto_scale_radio()
839
-
840
-
841
- def _default_pareto_scales():
842
- return [_PARETO_SCALE_DEFAULT] * MAX_PARETO_METRICS
843
-
844
-
845
- def _normalize_pareto_scales(scales):
846
- values = list(scales or [])
847
- if len(values) < MAX_PARETO_METRICS:
848
- values.extend(
849
- [_PARETO_SCALE_DEFAULT] * (MAX_PARETO_METRICS - len(values))
850
- )
851
- return values[:MAX_PARETO_METRICS]
852
-
853
-
854
- def _uniform_pareto_scales(scale):
855
- return [scale] * MAX_PARETO_METRICS
856
-
857
-
858
- def _pareto_master_scale_update(price_scales, time_scales):
859
- values = list(price_scales) + list(time_scales)
860
- if values and all(value == values[0] for value in values):
861
- return gr.update(value=values[0])
862
- return gr.update(value=None)
863
-
864
-
865
  def _pareto_axis(data, score_column, x_column, x_title, missing_message, empty_message, **hover):
866
  if x_column not in data.columns:
867
  return None, missing_message
@@ -877,12 +789,7 @@ def _pareto_axis(data, score_column, x_column, x_title, missing_message, empty_m
877
  return fig, None
878
 
879
 
880
- def _pareto_pair(
881
- data,
882
- score_column,
883
- latency_scale=_PARETO_SCALE_DEFAULT,
884
- price_scale=_PARETO_SCALE_DEFAULT,
885
- ):
886
  score_missing = "No score data is available for this metric."
887
  if data is None or not score_column or score_column not in data.columns:
888
  return None, score_missing, None, score_missing
@@ -895,7 +802,6 @@ def _pareto_pair(
895
  "Price per image isn't available for this dataset.",
896
  "No models have both a score and a price for this metric.",
897
  x_hover_prefix="$",
898
- x_axis_type=_pareto_axis_type(price_scale),
899
  )
900
  time_fig, time_message = _pareto_axis(
901
  data,
@@ -905,7 +811,6 @@ def _pareto_pair(
905
  "Min generation time isn't available for this dataset.",
906
  "No models have both a score and a min generation time for this metric.",
907
  x_hover_suffix="s",
908
- x_axis_type=_pareto_axis_type(latency_scale),
909
  )
910
  return price_fig, price_message, time_fig, time_message
911
 
@@ -944,16 +849,9 @@ def _pareto_slot_note(price_fig, price_message, time_fig, time_message, data):
944
  return " ".join(notes)
945
 
946
 
947
- def _pareto_slot_updates(
948
- data,
949
- score_columns,
950
- price_scales=None,
951
- time_scales=None,
952
- ):
953
  """Updates for a fixed bank of Gradio Plot slots (visible/hidden)."""
954
  score_columns = [column for column in (score_columns or []) if column]
955
- price_scales = _normalize_pareto_scales(price_scales)
956
- time_scales = _normalize_pareto_scales(time_scales)
957
  has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
958
  has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
959
  dataset_note = _pareto_dataset_message(data)
@@ -975,10 +873,7 @@ def _pareto_slot_updates(
975
  continue
976
  score_column = score_columns[index]
977
  price_fig, price_message, time_fig, time_message = _pareto_pair(
978
- data,
979
- score_column,
980
- latency_scale=time_scales[index],
981
- price_scale=price_scales[index],
982
  )
983
  show_price = price_fig is not None
984
  show_time = time_fig is not None
@@ -1005,30 +900,6 @@ def _pareto_slot_updates(
1005
  return updates
1006
 
1007
 
1008
- def _pareto_all_scale_updates(data, score_columns, scale):
1009
- """Apply one scale to every Pareto plot and radio."""
1010
- score_columns = [column for column in (score_columns or []) if column]
1011
- price_updates = []
1012
- time_updates = []
1013
- for index in range(MAX_PARETO_METRICS):
1014
- if index >= len(score_columns):
1015
- price_updates.append(gr.skip())
1016
- time_updates.append(gr.skip())
1017
- continue
1018
- price_fig, _, time_fig, _ = _pareto_pair(
1019
- data,
1020
- score_columns[index],
1021
- latency_scale=scale,
1022
- price_scale=scale,
1023
- )
1024
- price_updates.append(_pareto_plot_update(price_fig))
1025
- time_updates.append(_pareto_plot_update(time_fig))
1026
- radio_updates = [
1027
- gr.update(value=scale) for _ in range(MAX_PARETO_METRICS * 2)
1028
- ]
1029
- return price_updates + time_updates + radio_updates
1030
-
1031
-
1032
  def _samples_html(samples, selected_models, num_prompts, seed=0):
1033
  if not samples:
1034
  return _pareto_unavailable_html(
@@ -1173,10 +1044,10 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1173
  gr.Markdown(
1174
  "<p class='filter-help'>"
1175
  "These filters apply to Leaderboards, Pareto plots, and Samples. "
1176
- "On Samples, only datasets and models we have generations for "
1177
- "are listed. On Pareto plots, only datasets with price or "
1178
- "generation time are listed. Search in Models, or leave it "
1179
- "empty to include every model."
1180
  "</p>",
1181
  elem_classes="filter-help-host",
1182
  )
@@ -1246,27 +1117,12 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1246
  ) as pp_tab:
1247
  gr.Markdown(
1248
  "<p class='view-help'>"
1249
- "Score against price and generation time. Green points are on "
1250
- "the frontier; lavender points sit below it. Hover a point to "
1251
- "see which model it is."
1252
- "</p>"
1253
- "<p class='view-help'>"
1254
- "Plots use a logarithmic scale by default. You can switch "
1255
- "to linear for all plots, or individually for each plot."
1256
  "</p>",
1257
  elem_classes="view-help-host",
1258
  )
1259
- with gr.Row(
1260
- equal_height=False,
1261
- elem_classes="pareto-scale-all-row",
1262
- ):
1263
- gr.HTML(
1264
- "<span class='pareto-scale-all-label'>All plots</span>",
1265
- padding=False,
1266
- )
1267
- pareto_all_scale = _pareto_scale_radio(
1268
- "pareto-scale-toggle-all",
1269
- )
1270
  pareto_dataset_note = gr.HTML(
1271
  "",
1272
  padding=False,
@@ -1292,8 +1148,9 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1292
  min_width=320,
1293
  elem_classes="pareto-col",
1294
  ) as slot_price_col:
1295
- slot_price_scale = _pareto_plot_heading(
1296
- "Price vs score"
 
1297
  )
1298
  slot_price = gr.Plot(
1299
  value=None,
@@ -1305,8 +1162,9 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1305
  min_width=320,
1306
  elem_classes="pareto-col",
1307
  ) as slot_time_col:
1308
- slot_time_scale = _pareto_plot_heading(
1309
- "Min generation time vs score"
 
1310
  )
1311
  slot_time = gr.Plot(
1312
  value=None,
@@ -1327,10 +1185,8 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1327
  slot_layout,
1328
  slot_price_col,
1329
  slot_price,
1330
- slot_price_scale,
1331
  slot_time_col,
1332
  slot_time,
1333
- slot_time_scale,
1334
  )
1335
  )
1336
 
@@ -1379,16 +1235,12 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1379
  with gr.TabItem("About", id=TAB_ABOUT) as about_tab:
1380
  render_about()
1381
 
1382
- def _synced_filters(
1383
- dataset_id, metric_id, models, *, clear_metric=False, require_samples=False
1384
- ):
1385
  if clear_metric:
1386
  metric_id = []
1387
  else:
1388
  metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1389
- model_choices = _model_choices(
1390
- datasets, dataset_id, require_samples=require_samples
1391
- )
1392
  model_values = set(_model_choice_values(model_choices))
1393
  models = [model for model in (models or []) if model in model_values]
1394
  metric_choices = _metric_dropdown_choices(datasets, metrics, dataset_id)
@@ -1472,8 +1324,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1472
  "optimized": list(
1473
  extras.get("optimized", prev.get("optimized") or [])
1474
  ),
1475
- "price_scales": _normalize_pareto_scales(prev.get("price_scales")),
1476
- "time_scales": _normalize_pareto_scales(prev.get("time_scales")),
1477
  "stale": {
1478
  TAB_LEADERBOARDS: not flags["include_leaderboard"],
1479
  TAB_PARETO: not flags["include_pareto"],
@@ -1533,8 +1383,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1533
  include_leaderboard=True,
1534
  include_pareto=False,
1535
  include_samples=False,
1536
- price_scales=None,
1537
- time_scales=None,
1538
  ):
1539
  view = resolve_view(datasets, metrics, dataset_id, metric_id)
1540
  data = view["data"]
@@ -1555,12 +1403,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1555
  ranking_html = gr.skip()
1556
  if include_pareto:
1557
  pareto_data = _filter_leaderboard(data, [], [], [], models=models)
1558
- pareto_updates = _pareto_slot_updates(
1559
- pareto_data,
1560
- view["score_columns"],
1561
- price_scales=price_scales,
1562
- time_scales=time_scales,
1563
- )
1564
  else:
1565
  pareto_updates = _pareto_skip_updates()
1566
  if include_samples:
@@ -1602,20 +1445,10 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1602
  dataset_changed = source == "dataset" and dataset_id != view_state.get(
1603
  "dataset_id"
1604
  )
1605
- can_pareto = _dataset_has_pareto(datasets, dataset_id)
1606
- can_samples = _dataset_has_samples(datasets, dataset_id)
1607
- selected_tab = tab
1608
  if source == "dataset":
1609
- if tab == TAB_SAMPLES and not can_samples:
1610
- selected_tab = TAB_LEADERBOARDS
1611
- elif tab == TAB_PARETO and not can_pareto:
1612
- selected_tab = TAB_LEADERBOARDS
1613
  synced = _synced_filters(
1614
- dataset_id,
1615
- metric_id,
1616
- models,
1617
- clear_metric=dataset_changed,
1618
- require_samples=selected_tab == TAB_SAMPLES,
1619
  )
1620
  dataset_id, metric_id, models = synced[:3]
1621
  metric_update, models_update = synced[3], synced[4]
@@ -1653,13 +1486,20 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1653
  ):
1654
  return None
1655
 
 
1656
  extras = (
1657
  list(platform_value or []),
1658
  list(owner_value or []),
1659
  list(optimized_value or []),
1660
  )
1661
  extra_updates = None
 
 
1662
  if source == "dataset":
 
 
 
 
1663
  view = resolve_view(datasets, metrics, dataset_id, metric_id)
1664
  extra_updates = _leaderboard_extras(
1665
  view["data"] if view else None,
@@ -1697,8 +1537,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1697
  extras[2],
1698
  num_prompts,
1699
  seed,
1700
- price_scales=view_state.get("price_scales"),
1701
- time_scales=view_state.get("time_scales"),
1702
  **flags,
1703
  ),
1704
  "state": _commit_state(
@@ -1839,17 +1677,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1839
  )
1840
  dataset_update = _dataset_dropdown_update(datasets, tab, dataset_id)
1841
  metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1842
- require_samples = tab == TAB_SAMPLES
1843
- model_choices = _model_choices(
1844
- datasets, dataset_id, require_samples=require_samples
1845
- )
1846
- allowed_models = set(_model_choice_values(model_choices))
1847
- models = [model for model in (models or []) if model in allowed_models]
1848
- models_update = (
1849
- gr.update(choices=model_choices, value=models)
1850
- if (prev_tab == TAB_SAMPLES) != require_samples
1851
- else gr.skip()
1852
- )
1853
  view_state["current_tab"] = tab
1854
  view_state["dataset_id"] = dataset_id
1855
  view_state["metric_id"] = metric_id
@@ -1886,7 +1714,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1886
  filters_vis,
1887
  dataset_update,
1888
  metric_vis,
1889
- models_update,
1890
  *lb_filters,
1891
  )
1892
  tab_select = (
@@ -1911,8 +1739,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1911
  optimized_value,
1912
  num_prompts,
1913
  seed,
1914
- price_scales=view_state.get("price_scales"),
1915
- time_scales=view_state.get("time_scales"),
1916
  **flags,
1917
  )
1918
  stale[tab] = False
@@ -1969,70 +1795,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1969
  next_seed,
1970
  )
1971
 
1972
- def _on_pareto_plot_scale(slot_index, axis):
1973
- def handler(dataset_id, metric_id, models, scale, view_state):
1974
- view_state = dict(view_state or {})
1975
- price_scales = _normalize_pareto_scales(
1976
- view_state.get("price_scales")
1977
- )
1978
- time_scales = _normalize_pareto_scales(
1979
- view_state.get("time_scales")
1980
- )
1981
- if axis == "price":
1982
- if price_scales[slot_index] == scale:
1983
- return gr.skip(), gr.skip(), gr.skip()
1984
- price_scales[slot_index] = scale
1985
- else:
1986
- if time_scales[slot_index] == scale:
1987
- return gr.skip(), gr.skip(), gr.skip()
1988
- time_scales[slot_index] = scale
1989
- view_state["price_scales"] = price_scales
1990
- view_state["time_scales"] = time_scales
1991
- master_scale = _pareto_master_scale_update(
1992
- price_scales, time_scales
1993
- )
1994
- view = resolve_view(datasets, metrics, dataset_id, metric_id)
1995
- score_columns = [
1996
- column for column in (view["score_columns"] or []) if column
1997
- ]
1998
- if slot_index >= len(score_columns):
1999
- return gr.skip(), master_scale, view_state
2000
- data = _filter_leaderboard(
2001
- view["data"], [], [], [], models=list(models or [])
2002
- )
2003
- price_fig, _, time_fig, _ = _pareto_pair(
2004
- data,
2005
- score_columns[slot_index],
2006
- latency_scale=time_scales[slot_index],
2007
- price_scale=price_scales[slot_index],
2008
- )
2009
- fig = price_fig if axis == "price" else time_fig
2010
- return _pareto_plot_update(fig), master_scale, view_state
2011
-
2012
- handler.__name__ = f"on_pareto_{axis}_scale_{slot_index}"
2013
- return handler
2014
-
2015
- def on_pareto_all_scale(dataset_id, metric_id, models, scale, view_state):
2016
- if scale not in _PARETO_SCALE_VALUES:
2017
- return (*_skip_all(MAX_PARETO_METRICS * 4), gr.skip())
2018
- view_state = dict(view_state or {})
2019
- scales = _uniform_pareto_scales(scale)
2020
- if (
2021
- _normalize_pareto_scales(view_state.get("price_scales")) == scales
2022
- and _normalize_pareto_scales(view_state.get("time_scales")) == scales
2023
- ):
2024
- return (*_skip_all(MAX_PARETO_METRICS * 4), gr.skip())
2025
- view_state["price_scales"] = scales
2026
- view_state["time_scales"] = scales
2027
- view = resolve_view(datasets, metrics, dataset_id, metric_id)
2028
- data = _filter_leaderboard(
2029
- view["data"], [], [], [], models=list(models or [])
2030
- )
2031
- return (
2032
- *_pareto_all_scale_updates(data, view["score_columns"], scale),
2033
- view_state,
2034
- )
2035
-
2036
  def _on_tab(tab):
2037
  def handler(
2038
  dataset_id,
@@ -2070,8 +1832,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
2070
  "platform": [],
2071
  "owner": [],
2072
  "optimized": [],
2073
- "price_scales": _default_pareto_scales(),
2074
- "time_scales": _default_pareto_scales(),
2075
  "stale": {
2076
  TAB_LEADERBOARDS: False,
2077
  TAB_PARETO: True,
@@ -2083,7 +1843,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
2083
  pareto_dataset_note,
2084
  *[
2085
  component
2086
- for slot_group, slot_title, slot_note, slot_layout, slot_price_col, slot_price, slot_price_scale, slot_time_col, slot_time, slot_time_scale in pareto_slots
2087
  for component in (
2088
  slot_group,
2089
  slot_title,
@@ -2096,24 +1856,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
2096
  )
2097
  ],
2098
  ]
2099
- pareto_all_scale_outputs = [
2100
- *[
2101
- slot_price
2102
- for _, _, _, _, _, slot_price, _, _, _, _ in pareto_slots
2103
- ],
2104
- *[
2105
- slot_time
2106
- for _, _, _, _, _, _, _, _, slot_time, _ in pareto_slots
2107
- ],
2108
- *[
2109
- slot_price_scale
2110
- for _, _, _, _, _, _, slot_price_scale, _, _, _ in pareto_slots
2111
- ],
2112
- *[
2113
- slot_time_scale
2114
- for _, _, _, _, _, _, _, _, _, slot_time_scale in pareto_slots
2115
- ],
2116
- ]
2117
  view_inputs = [
2118
  platform,
2119
  owner,
@@ -2218,56 +1960,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
2218
  show_progress="hidden",
2219
  )
2220
 
2221
- pareto_all_scale.change(
2222
- on_pareto_all_scale,
2223
- inputs=[
2224
- dataset_dd,
2225
- metric_dd,
2226
- models_dd,
2227
- pareto_all_scale,
2228
- view_state,
2229
- ],
2230
- outputs=[*pareto_all_scale_outputs, view_state],
2231
- **_VIEW_EVENTS,
2232
- )
2233
-
2234
- for slot_index, (
2235
- _,
2236
- _,
2237
- _,
2238
- _,
2239
- _,
2240
- slot_price,
2241
- slot_price_scale,
2242
- _,
2243
- slot_time,
2244
- slot_time_scale,
2245
- ) in enumerate(pareto_slots):
2246
- slot_price_scale.change(
2247
- _on_pareto_plot_scale(slot_index, "price"),
2248
- inputs=[
2249
- dataset_dd,
2250
- metric_dd,
2251
- models_dd,
2252
- slot_price_scale,
2253
- view_state,
2254
- ],
2255
- outputs=[slot_price, pareto_all_scale, view_state],
2256
- **_VIEW_EVENTS,
2257
- )
2258
- slot_time_scale.change(
2259
- _on_pareto_plot_scale(slot_index, "time"),
2260
- inputs=[
2261
- dataset_dd,
2262
- metric_dd,
2263
- models_dd,
2264
- slot_time_scale,
2265
- view_state,
2266
- ],
2267
- outputs=[slot_time, pareto_all_scale, view_state],
2268
- **_VIEW_EVENTS,
2269
- )
2270
-
2271
  prompt_count.change(
2272
  on_samples_controls,
2273
  inputs=[dataset_dd, models_dd, prompt_count, seed_state],
 
1
  from html import escape
 
2
  from pathlib import Path
3
  import base64
4
  import random
 
25
  _PARETO_SLOT_COUNT = 1 + MAX_PARETO_METRICS * 8
26
  _PARETO_PRICE_COLUMN = "Price / Image (USD)"
27
  _PARETO_TIME_COLUMN = "Min Generation Time (s)"
 
 
 
 
 
 
28
 
29
  TAB_LEADERBOARDS = "leaderboards"
30
  TAB_PARETO = "pareto"
 
244
  return bool(dataset and dataset.get("samples"))
245
 
246
 
 
 
 
 
 
 
 
 
247
  def _dataset_has_pareto(datasets, dataset_id):
248
  dataset = _item(datasets, dataset_id)
249
  columns = getattr(dataset.get("data") if dataset else None, "columns", [])
 
314
  ]
315
 
316
 
317
+ def _model_choices(datasets, dataset_id):
318
  cached = _MODEL_CHOICES_CACHE.get(dataset_id)
319
+ if cached is not None:
 
 
 
 
 
 
 
 
 
 
 
 
 
320
  return cached
321
+ dataset = _item(datasets, dataset_id)
322
+ data = dataset.get("data") if dataset else None
323
+ if data is None or "Model" not in getattr(data, "columns", []):
324
+ _MODEL_CHOICES_CACHE[dataset_id] = []
325
+ return []
326
+ models = data["Model"].dropna().astype(str).unique().tolist()
327
+ # (label, value) so the UI shows the shared name but filters on the raw id.
328
+ choices = sorted(
329
+ ((display_model_name(model), model) for model in models),
330
+ key=lambda item: item[0].casefold(),
331
+ )
332
+ _MODEL_CHOICES_CACHE[dataset_id] = choices
333
+ return choices
334
 
335
 
336
  def _model_choice_values(choices):
 
585
  "Arena Art Elo": "Art",
586
  "Arena Portraits Elo": "Portraits",
587
  "Arena Text Rendering Elo": "Text Rendering",
588
+ "Raw Win Rate": "Raw win rate",
589
  "Median Generation Time (s)": "Median generation time",
590
  "Min Generation Time (s)": "Min generation time",
591
  "Price / Image (USD)": "Price per image",
 
670
  x_title,
671
  x_hover_prefix="",
672
  x_hover_suffix="",
 
673
  ):
674
  scatter = (
675
  data[["Model", score_column, x_column]]
 
755
  )
756
  axis_font = {"color": "#fafafa", "size": 13}
757
  tick_font = {"color": "#a3a3a3", "size": 12}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
758
  fig.update_xaxes(
 
 
759
  showgrid=True,
760
  gridcolor="rgba(74, 57, 98, 0.55)",
761
  zeroline=False,
 
774
  return fig
775
 
776
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
777
  def _pareto_axis(data, score_column, x_column, x_title, missing_message, empty_message, **hover):
778
  if x_column not in data.columns:
779
  return None, missing_message
 
789
  return fig, None
790
 
791
 
792
+ def _pareto_pair(data, score_column):
 
 
 
 
 
793
  score_missing = "No score data is available for this metric."
794
  if data is None or not score_column or score_column not in data.columns:
795
  return None, score_missing, None, score_missing
 
802
  "Price per image isn't available for this dataset.",
803
  "No models have both a score and a price for this metric.",
804
  x_hover_prefix="$",
 
805
  )
806
  time_fig, time_message = _pareto_axis(
807
  data,
 
811
  "Min generation time isn't available for this dataset.",
812
  "No models have both a score and a min generation time for this metric.",
813
  x_hover_suffix="s",
 
814
  )
815
  return price_fig, price_message, time_fig, time_message
816
 
 
849
  return " ".join(notes)
850
 
851
 
852
+ def _pareto_slot_updates(data, score_columns):
 
 
 
 
 
853
  """Updates for a fixed bank of Gradio Plot slots (visible/hidden)."""
854
  score_columns = [column for column in (score_columns or []) if column]
 
 
855
  has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
856
  has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
857
  dataset_note = _pareto_dataset_message(data)
 
873
  continue
874
  score_column = score_columns[index]
875
  price_fig, price_message, time_fig, time_message = _pareto_pair(
876
+ data, score_column
 
 
 
877
  )
878
  show_price = price_fig is not None
879
  show_time = time_fig is not None
 
900
  return updates
901
 
902
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
903
  def _samples_html(samples, selected_models, num_prompts, seed=0):
904
  if not samples:
905
  return _pareto_unavailable_html(
 
1044
  gr.Markdown(
1045
  "<p class='filter-help'>"
1046
  "These filters apply to Leaderboards, Pareto plots, and Samples. "
1047
+ "On Samples, only datasets we have generations for are listed. "
1048
+ "On Pareto plots, only datasets with price or generation time "
1049
+ "are listed. Search in Models, or leave it empty to include "
1050
+ "every model."
1051
  "</p>",
1052
  elem_classes="filter-help-host",
1053
  )
 
1117
  ) as pp_tab:
1118
  gr.Markdown(
1119
  "<p class='view-help'>"
1120
+ "Score against price and generation time. Green points are on the "
1121
+ "frontier; lavender points sit below it. Hover a point to see "
1122
+ "which model it is."
 
 
 
 
1123
  "</p>",
1124
  elem_classes="view-help-host",
1125
  )
 
 
 
 
 
 
 
 
 
 
 
1126
  pareto_dataset_note = gr.HTML(
1127
  "",
1128
  padding=False,
 
1148
  min_width=320,
1149
  elem_classes="pareto-col",
1150
  ) as slot_price_col:
1151
+ gr.Markdown(
1152
+ "#### Price vs score",
1153
+ elem_classes="pareto-subhead",
1154
  )
1155
  slot_price = gr.Plot(
1156
  value=None,
 
1162
  min_width=320,
1163
  elem_classes="pareto-col",
1164
  ) as slot_time_col:
1165
+ gr.Markdown(
1166
+ "#### Min generation time vs score",
1167
+ elem_classes="pareto-subhead",
1168
  )
1169
  slot_time = gr.Plot(
1170
  value=None,
 
1185
  slot_layout,
1186
  slot_price_col,
1187
  slot_price,
 
1188
  slot_time_col,
1189
  slot_time,
 
1190
  )
1191
  )
1192
 
 
1235
  with gr.TabItem("About", id=TAB_ABOUT) as about_tab:
1236
  render_about()
1237
 
1238
+ def _synced_filters(dataset_id, metric_id, models, *, clear_metric=False):
 
 
1239
  if clear_metric:
1240
  metric_id = []
1241
  else:
1242
  metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1243
+ model_choices = _model_choices(datasets, dataset_id)
 
 
1244
  model_values = set(_model_choice_values(model_choices))
1245
  models = [model for model in (models or []) if model in model_values]
1246
  metric_choices = _metric_dropdown_choices(datasets, metrics, dataset_id)
 
1324
  "optimized": list(
1325
  extras.get("optimized", prev.get("optimized") or [])
1326
  ),
 
 
1327
  "stale": {
1328
  TAB_LEADERBOARDS: not flags["include_leaderboard"],
1329
  TAB_PARETO: not flags["include_pareto"],
 
1383
  include_leaderboard=True,
1384
  include_pareto=False,
1385
  include_samples=False,
 
 
1386
  ):
1387
  view = resolve_view(datasets, metrics, dataset_id, metric_id)
1388
  data = view["data"]
 
1403
  ranking_html = gr.skip()
1404
  if include_pareto:
1405
  pareto_data = _filter_leaderboard(data, [], [], [], models=models)
1406
+ pareto_updates = _pareto_slot_updates(pareto_data, view["score_columns"])
 
 
 
 
 
1407
  else:
1408
  pareto_updates = _pareto_skip_updates()
1409
  if include_samples:
 
1445
  dataset_changed = source == "dataset" and dataset_id != view_state.get(
1446
  "dataset_id"
1447
  )
1448
+
 
 
1449
  if source == "dataset":
 
 
 
 
1450
  synced = _synced_filters(
1451
+ dataset_id, metric_id, models, clear_metric=dataset_changed
 
 
 
 
1452
  )
1453
  dataset_id, metric_id, models = synced[:3]
1454
  metric_update, models_update = synced[3], synced[4]
 
1486
  ):
1487
  return None
1488
 
1489
+ selected_tab = tab
1490
  extras = (
1491
  list(platform_value or []),
1492
  list(owner_value or []),
1493
  list(optimized_value or []),
1494
  )
1495
  extra_updates = None
1496
+ can_pareto = _dataset_has_pareto(datasets, dataset_id)
1497
+ can_samples = _dataset_has_samples(datasets, dataset_id)
1498
  if source == "dataset":
1499
+ if tab == TAB_SAMPLES and not can_samples:
1500
+ selected_tab = TAB_LEADERBOARDS
1501
+ elif tab == TAB_PARETO and not can_pareto:
1502
+ selected_tab = TAB_LEADERBOARDS
1503
  view = resolve_view(datasets, metrics, dataset_id, metric_id)
1504
  extra_updates = _leaderboard_extras(
1505
  view["data"] if view else None,
 
1537
  extras[2],
1538
  num_prompts,
1539
  seed,
 
 
1540
  **flags,
1541
  ),
1542
  "state": _commit_state(
 
1677
  )
1678
  dataset_update = _dataset_dropdown_update(datasets, tab, dataset_id)
1679
  metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1680
+ models = list(models or [])
 
 
 
 
 
 
 
 
 
 
1681
  view_state["current_tab"] = tab
1682
  view_state["dataset_id"] = dataset_id
1683
  view_state["metric_id"] = metric_id
 
1714
  filters_vis,
1715
  dataset_update,
1716
  metric_vis,
1717
+ gr.skip(),
1718
  *lb_filters,
1719
  )
1720
  tab_select = (
 
1739
  optimized_value,
1740
  num_prompts,
1741
  seed,
 
 
1742
  **flags,
1743
  )
1744
  stale[tab] = False
 
1795
  next_seed,
1796
  )
1797
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1798
  def _on_tab(tab):
1799
  def handler(
1800
  dataset_id,
 
1832
  "platform": [],
1833
  "owner": [],
1834
  "optimized": [],
 
 
1835
  "stale": {
1836
  TAB_LEADERBOARDS: False,
1837
  TAB_PARETO: True,
 
1843
  pareto_dataset_note,
1844
  *[
1845
  component
1846
+ for slot_group, slot_title, slot_note, slot_layout, slot_price_col, slot_price, slot_time_col, slot_time in pareto_slots
1847
  for component in (
1848
  slot_group,
1849
  slot_title,
 
1856
  )
1857
  ],
1858
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1859
  view_inputs = [
1860
  platform,
1861
  owner,
 
1960
  show_progress="hidden",
1961
  )
1962
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1963
  prompt_count.change(
1964
  on_samples_controls,
1965
  inputs=[dataset_dd, models_dd, prompt_count, seed_state],