Spaces:
Running
Running
log scale
#2
by minette-kaunismaki - opened
- .gitignore +1 -5
- README.md +2 -0
- app.py +26 -299
- data/qwen_image_bench_model_price_and_median_generation_time.csv +60 -0
- data/qwen_image_bench_model_price_and_median_generation_time_10_august.csv +0 -64
- model_display.py +0 -1
- ui.py +47 -355
.gitignore
CHANGED
|
@@ -176,8 +176,4 @@ cython_debug/
|
|
| 176 |
evaluation_results/
|
| 177 |
images/
|
| 178 |
hf_cache/
|
| 179 |
-
*.lock
|
| 180 |
-
|
| 181 |
-
# macOS
|
| 182 |
-
.DS_Store
|
| 183 |
-
|
|
|
|
| 176 |
evaluation_results/
|
| 177 |
images/
|
| 178 |
hf_cache/
|
| 179 |
+
*.lock
|
|
|
|
|
|
|
|
|
|
|
|
README.md
CHANGED
|
@@ -30,6 +30,8 @@ pip install "gradio==5.19.0" pandas -r requirements.txt
|
|
| 30 |
python app.py
|
| 31 |
```
|
| 32 |
|
|
|
|
|
|
|
| 33 |
`requirements.txt` lists Plotly. Gradio and pandas are required locally;
|
| 34 |
Hugging Face Spaces installs Gradio from the YAML `sdk_version` above.
|
| 35 |
|
|
|
|
| 30 |
python app.py
|
| 31 |
```
|
| 32 |
|
| 33 |
+
The app is served at `http://127.0.0.1:7860`.
|
| 34 |
+
|
| 35 |
`requirements.txt` lists Plotly. Gradio and pandas are required locally;
|
| 36 |
Hugging Face Spaces installs Gradio from the YAML `sdk_version` above.
|
| 37 |
|
app.py
CHANGED
|
@@ -62,9 +62,6 @@ custom_css = """
|
|
| 62 |
--pruna-accordion-bg: rgba(255, 255, 255, 0.02);
|
| 63 |
--pruna-accordion-border: rgba(216, 180, 254, 0.15);
|
| 64 |
--pruna-dropdown-hover: #2a1844;
|
| 65 |
-
--pruna-toggle-track: var(--pruna-bg-header);
|
| 66 |
-
--pruna-toggle-thumb: var(--pruna-bg-elevated);
|
| 67 |
-
--pruna-toggle-thumb-shadow: 0 1px 2px rgba(0, 0, 0, 0.45), inset 0 1px rgba(255, 255, 255, 0.06);
|
| 68 |
color-scheme: dark;
|
| 69 |
}
|
| 70 |
|
|
@@ -108,27 +105,13 @@ custom_css = """
|
|
| 108 |
--pruna-accordion-bg: var(--pruna-bg-card);
|
| 109 |
--pruna-accordion-border: var(--pruna-border);
|
| 110 |
--pruna-dropdown-hover: #f3e8ff;
|
| 111 |
-
--pruna-toggle-track: var(--pruna-bg-header);
|
| 112 |
-
--pruna-toggle-thumb: var(--pruna-bg-card);
|
| 113 |
-
--pruna-toggle-thumb-shadow: 0 1px 2px rgba(88, 28, 135, 0.12);
|
| 114 |
color-scheme: light;
|
| 115 |
}
|
| 116 |
|
| 117 |
-
html {
|
| 118 |
width: 100% !important;
|
| 119 |
max-width: 100% !important;
|
| 120 |
-
|
| 121 |
-
overflow-x: hidden;
|
| 122 |
-
overflow-y: auto;
|
| 123 |
-
-webkit-text-size-adjust: 100%;
|
| 124 |
-
text-size-adjust: 100%;
|
| 125 |
-
-webkit-tap-highlight-color: transparent;
|
| 126 |
-
}
|
| 127 |
-
body, gradio-app {
|
| 128 |
-
width: 100% !important;
|
| 129 |
-
max-width: 100% !important;
|
| 130 |
-
min-width: 0 !important;
|
| 131 |
-
overflow: visible;
|
| 132 |
-webkit-tap-highlight-color: transparent;
|
| 133 |
}
|
| 134 |
html, body, .gradio-container, .main {
|
|
@@ -151,25 +134,18 @@ button, a, label, input, select, textarea,
|
|
| 151 |
/* Subtle depth — not a marketing-site hero glow */
|
| 152 |
body, .gradio-container {
|
| 153 |
background-image: var(--pruna-glow) !important;
|
| 154 |
-
background-
|
| 155 |
-
background-attachment: scroll !important;
|
| 156 |
-
}
|
| 157 |
-
@media (min-width: 701px) and (hover: hover) and (pointer: fine) {
|
| 158 |
-
body, .gradio-container {
|
| 159 |
-
background-attachment: fixed !important;
|
| 160 |
-
}
|
| 161 |
}
|
| 162 |
|
| 163 |
.gradio-container {
|
| 164 |
width: 100% !important;
|
| 165 |
max-width: 1200px !important;
|
| 166 |
-
min-width: 0 !important;
|
| 167 |
margin: 0 auto !important;
|
| 168 |
padding-top: 0 !important;
|
| 169 |
padding-left: 20px !important;
|
| 170 |
padding-right: 20px !important;
|
| 171 |
box-sizing: border-box !important;
|
| 172 |
-
overflow-x:
|
| 173 |
}
|
| 174 |
.gradio-container .main,
|
| 175 |
.gradio-container .wrap,
|
|
@@ -190,7 +166,6 @@ body, .gradio-container {
|
|
| 190 |
.workspace-filters,
|
| 191 |
.view-filters {
|
| 192 |
max-width: 100% !important;
|
| 193 |
-
min-width: 0 !important;
|
| 194 |
}
|
| 195 |
|
| 196 |
/* —— App header (P-Bench only) —— */
|
|
@@ -325,9 +300,6 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 325 |
background: transparent !important;
|
| 326 |
box-shadow: none !important;
|
| 327 |
}
|
| 328 |
-
/* Flatten tabs so the bar sits above shared filters. display:contents is
|
| 329 |
-
the fallback; Safari can drop or mis-order those children, so browsers
|
| 330 |
-
with subgrid use the grid layout below instead. */
|
| 331 |
.workspace-shell > .tabs,
|
| 332 |
.workspace-shell > .main-tabs,
|
| 333 |
.workspace-shell > .block:not(.workspace-filters),
|
|
@@ -351,49 +323,12 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 351 |
margin: 0 0 16px !important;
|
| 352 |
justify-content: center !important;
|
| 353 |
width: 100% !important;
|
| 354 |
-
overflow: visible !important;
|
| 355 |
-
}
|
| 356 |
-
@supports (grid-template-rows: subgrid) {
|
| 357 |
-
.workspace-shell,
|
| 358 |
-
.workspace-shell.block,
|
| 359 |
-
.workspace-shell.column,
|
| 360 |
-
.workspace-shell.gap {
|
| 361 |
-
display: grid !important;
|
| 362 |
-
grid-template-columns: minmax(0, 1fr) !important;
|
| 363 |
-
grid-template-rows: auto auto auto !important;
|
| 364 |
-
align-content: start !important;
|
| 365 |
-
}
|
| 366 |
-
.workspace-shell > .tabs,
|
| 367 |
-
.workspace-shell > .main-tabs,
|
| 368 |
-
.workspace-shell .tabs.main-tabs {
|
| 369 |
-
display: grid !important;
|
| 370 |
-
grid-template-columns: minmax(0, 1fr) !important;
|
| 371 |
-
grid-template-rows: subgrid !important;
|
| 372 |
-
grid-column: 1 !important;
|
| 373 |
-
grid-row: 1 / 4 !important;
|
| 374 |
-
position: static !important;
|
| 375 |
-
}
|
| 376 |
-
.main-tabs > .tab-wrapper {
|
| 377 |
-
grid-row: 1 !important;
|
| 378 |
-
order: 0 !important;
|
| 379 |
-
}
|
| 380 |
-
.workspace-filters {
|
| 381 |
-
grid-column: 1 !important;
|
| 382 |
-
grid-row: 2 !important;
|
| 383 |
-
order: 0 !important;
|
| 384 |
-
}
|
| 385 |
-
.main-tabs .tabitem {
|
| 386 |
-
grid-row: 3 !important;
|
| 387 |
-
order: 0 !important;
|
| 388 |
-
min-width: 0 !important;
|
| 389 |
-
}
|
| 390 |
}
|
| 391 |
.main-tabs .tab-container {
|
| 392 |
height: auto !important;
|
| 393 |
-
min-height: 0 !important;
|
| 394 |
justify-content: center !important;
|
| 395 |
flex-wrap: wrap !important;
|
| 396 |
-
overflow:
|
| 397 |
max-width: 100% !important;
|
| 398 |
gap: 2px;
|
| 399 |
}
|
|
@@ -510,8 +445,6 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 510 |
.app-header-brand h1,
|
| 511 |
.gradio-container .app-header-brand h1 {
|
| 512 |
font-size: 1.55rem !important;
|
| 513 |
-
width: auto !important;
|
| 514 |
-
max-width: 100% !important;
|
| 515 |
}
|
| 516 |
.app-header-tagline {
|
| 517 |
font-size: 0.88rem !important;
|
|
@@ -588,29 +521,17 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 588 |
left: 0 !important;
|
| 589 |
width: 2.4rem;
|
| 590 |
min-width: 2.4rem;
|
| 591 |
-
box-shadow: 6px 0 8px -6px rgba(0, 0, 0, 0.45);
|
| 592 |
}
|
| 593 |
.ranking-table .model-cell,
|
| 594 |
-
.prose .ranking-table .model-cell
|
| 595 |
-
|
| 596 |
-
left: auto !important;
|
| 597 |
-
z-index: auto;
|
| 598 |
-
min-width: 140px;
|
| 599 |
-
max-width: none;
|
| 600 |
-
background: transparent !important;
|
| 601 |
-
}
|
| 602 |
-
.ranking-table th.model-cell,
|
| 603 |
-
.prose .ranking-table th.model-cell {
|
| 604 |
position: sticky !important;
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
min-width: 140px;
|
| 609 |
-
max-width: none;
|
| 610 |
-
background: var(--pruna-bg-header) !important;
|
| 611 |
}
|
| 612 |
-
.ranking-table
|
| 613 |
-
|
| 614 |
}
|
| 615 |
.compare-controls,
|
| 616 |
.compare-controls.row,
|
|
@@ -683,7 +604,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 683 |
.view-filters {
|
| 684 |
display: flex !important;
|
| 685 |
flex-wrap: wrap !important;
|
| 686 |
-
align-items:
|
| 687 |
gap: 12px !important;
|
| 688 |
margin: 0;
|
| 689 |
overflow: visible !important;
|
|
@@ -691,7 +612,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 691 |
.view-filters > div,
|
| 692 |
.view-filters > .block,
|
| 693 |
.view-filters > .form {
|
| 694 |
-
flex: 1 1 0
|
| 695 |
min-width: 0 !important;
|
| 696 |
}
|
| 697 |
.view-filters > .block,
|
|
@@ -843,10 +764,6 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 843 |
line-height: 1.45 !important;
|
| 844 |
font-weight: 400 !important;
|
| 845 |
}
|
| 846 |
-
.view-help + .view-help,
|
| 847 |
-
.prose .view-help + .view-help {
|
| 848 |
-
margin-top: 0.45rem !important;
|
| 849 |
-
}
|
| 850 |
.view-filters span[data-testid="block-info"],
|
| 851 |
.view-filters .info,
|
| 852 |
.view-filters .block-info {
|
|
@@ -984,7 +901,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 984 |
.leaderboard-controls {
|
| 985 |
display: flex !important;
|
| 986 |
flex-wrap: wrap !important;
|
| 987 |
-
align-items:
|
| 988 |
gap: 10px !important;
|
| 989 |
margin-bottom: 12px;
|
| 990 |
overflow: visible !important;
|
|
@@ -1400,9 +1317,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 1400 |
box-shadow: none !important;
|
| 1401 |
}
|
| 1402 |
.compare-controls .compare-prompt-count .head {
|
| 1403 |
-
display:
|
| 1404 |
-
grid-column: 1 / -1;
|
| 1405 |
-
grid-row: 1;
|
| 1406 |
margin: 0 !important;
|
| 1407 |
}
|
| 1408 |
.compare-controls .compare-prompt-count .head label {
|
|
@@ -1576,182 +1491,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 1576 |
margin: 0 !important;
|
| 1577 |
}
|
| 1578 |
.compare-row { display: grid; gap: 12px; min-width: 0; width: 100%; }
|
| 1579 |
-
.compare-prompt-text { overflow-wrap: anywhere;
|
| 1580 |
-
.pareto-heading-row,
|
| 1581 |
-
.pareto-heading-row.row,
|
| 1582 |
-
.pareto-heading-row .form {
|
| 1583 |
-
display: flex !important;
|
| 1584 |
-
flex-wrap: wrap !important;
|
| 1585 |
-
align-items: center !important;
|
| 1586 |
-
gap: 8px 12px !important;
|
| 1587 |
-
width: 100% !important;
|
| 1588 |
-
margin-bottom: 0.4rem !important;
|
| 1589 |
-
}
|
| 1590 |
-
.pareto-heading-row .pareto-subhead,
|
| 1591 |
-
.pareto-heading-row > div:first-child,
|
| 1592 |
-
.pareto-heading-row .form > div:first-child {
|
| 1593 |
-
flex: 1 1 240px !important;
|
| 1594 |
-
min-width: 0 !important;
|
| 1595 |
-
margin: 0 !important;
|
| 1596 |
-
}
|
| 1597 |
-
.pareto-heading-row .pareto-scale-control {
|
| 1598 |
-
display: flex !important;
|
| 1599 |
-
flex-direction: row !important;
|
| 1600 |
-
align-items: center !important;
|
| 1601 |
-
justify-content: flex-end !important;
|
| 1602 |
-
flex: 0 0 auto !important;
|
| 1603 |
-
gap: 0 !important;
|
| 1604 |
-
margin-left: auto !important;
|
| 1605 |
-
max-width: 168px !important;
|
| 1606 |
-
padding: 0 !important;
|
| 1607 |
-
}
|
| 1608 |
-
.pareto-scale-all-row,
|
| 1609 |
-
.pareto-scale-all-row.row,
|
| 1610 |
-
.pareto-scale-all-row .form {
|
| 1611 |
-
display: flex !important;
|
| 1612 |
-
flex-direction: row !important;
|
| 1613 |
-
flex-wrap: wrap !important;
|
| 1614 |
-
align-items: center !important;
|
| 1615 |
-
justify-content: flex-start !important;
|
| 1616 |
-
gap: 8px 12px !important;
|
| 1617 |
-
width: 100% !important;
|
| 1618 |
-
margin: 2px 0 14px !important;
|
| 1619 |
-
}
|
| 1620 |
-
.pareto-scale-all-row .html-container,
|
| 1621 |
-
.pareto-scale-all-row .block {
|
| 1622 |
-
border: none !important;
|
| 1623 |
-
background: transparent !important;
|
| 1624 |
-
box-shadow: none !important;
|
| 1625 |
-
padding: 0 !important;
|
| 1626 |
-
margin: 0 !important;
|
| 1627 |
-
width: auto !important;
|
| 1628 |
-
flex: 0 0 auto !important;
|
| 1629 |
-
}
|
| 1630 |
-
.pareto-scale-all-row .html-container {
|
| 1631 |
-
flex: 1 1 auto !important;
|
| 1632 |
-
min-width: 0 !important;
|
| 1633 |
-
}
|
| 1634 |
-
.pareto-scale-all-label {
|
| 1635 |
-
color: var(--pruna-text-muted);
|
| 1636 |
-
font-size: 0.8rem;
|
| 1637 |
-
font-weight: 500;
|
| 1638 |
-
white-space: nowrap;
|
| 1639 |
-
}
|
| 1640 |
-
.pareto-scale-all-row .pareto-scale-toggle {
|
| 1641 |
-
margin-left: auto !important;
|
| 1642 |
-
}
|
| 1643 |
-
.pareto-scale-toggle,
|
| 1644 |
-
.pareto-scale-toggle.block {
|
| 1645 |
-
min-width: 0 !important;
|
| 1646 |
-
width: auto !important;
|
| 1647 |
-
border: none !important;
|
| 1648 |
-
background: transparent !important;
|
| 1649 |
-
box-shadow: none !important;
|
| 1650 |
-
padding: 0 !important;
|
| 1651 |
-
margin: 0 !important;
|
| 1652 |
-
}
|
| 1653 |
-
.pareto-scale-toggle .wrap,
|
| 1654 |
-
.pareto-scale-toggle .form {
|
| 1655 |
-
display: block !important;
|
| 1656 |
-
width: auto !important;
|
| 1657 |
-
margin: 0 !important;
|
| 1658 |
-
padding: 0 !important;
|
| 1659 |
-
border: none !important;
|
| 1660 |
-
background: transparent !important;
|
| 1661 |
-
box-shadow: none !important;
|
| 1662 |
-
}
|
| 1663 |
-
.pareto-scale-toggle legend {
|
| 1664 |
-
display: none !important;
|
| 1665 |
-
}
|
| 1666 |
-
.pareto-scale-toggle fieldset,
|
| 1667 |
-
.pareto-scale-toggle .wrap:has(> label),
|
| 1668 |
-
.pareto-scale-toggle .form:has(> label) {
|
| 1669 |
-
position: relative !important;
|
| 1670 |
-
display: grid !important;
|
| 1671 |
-
grid-template-columns: 1fr 1fr !important;
|
| 1672 |
-
align-items: stretch !important;
|
| 1673 |
-
isolation: isolate;
|
| 1674 |
-
box-sizing: border-box !important;
|
| 1675 |
-
width: max-content !important;
|
| 1676 |
-
min-width: 0 !important;
|
| 1677 |
-
padding: 4px !important;
|
| 1678 |
-
gap: 4px !important;
|
| 1679 |
-
border: 1px solid var(--pruna-input-border) !important;
|
| 1680 |
-
border-radius: 10px !important;
|
| 1681 |
-
background: var(--pruna-toggle-track) !important;
|
| 1682 |
-
box-shadow: none !important;
|
| 1683 |
-
}
|
| 1684 |
-
.pareto-scale-toggle fieldset::before,
|
| 1685 |
-
.pareto-scale-toggle .wrap:has(> label)::before,
|
| 1686 |
-
.pareto-scale-toggle .form:has(> label)::before {
|
| 1687 |
-
content: none !important;
|
| 1688 |
-
}
|
| 1689 |
-
.pareto-scale-toggle label {
|
| 1690 |
-
position: relative !important;
|
| 1691 |
-
z-index: 1 !important;
|
| 1692 |
-
display: flex !important;
|
| 1693 |
-
flex: 1 1 auto !important;
|
| 1694 |
-
align-items: center !important;
|
| 1695 |
-
justify-content: center !important;
|
| 1696 |
-
gap: 0 !important;
|
| 1697 |
-
box-sizing: border-box !important;
|
| 1698 |
-
min-width: 58px !important;
|
| 1699 |
-
min-height: 26px !important;
|
| 1700 |
-
margin: 0 !important;
|
| 1701 |
-
padding: 5px 12px !important;
|
| 1702 |
-
border: none !important;
|
| 1703 |
-
border-radius: 6px !important;
|
| 1704 |
-
background: transparent !important;
|
| 1705 |
-
box-shadow: none !important;
|
| 1706 |
-
color: var(--pruna-text-body) !important;
|
| 1707 |
-
font-size: 0.72rem !important;
|
| 1708 |
-
font-weight: 600 !important;
|
| 1709 |
-
line-height: 1.2 !important;
|
| 1710 |
-
letter-spacing: 0.01em;
|
| 1711 |
-
white-space: nowrap;
|
| 1712 |
-
cursor: pointer !important;
|
| 1713 |
-
}
|
| 1714 |
-
.pareto-scale-toggle-all label {
|
| 1715 |
-
min-width: 68px !important;
|
| 1716 |
-
min-height: 30px !important;
|
| 1717 |
-
padding: 6px 14px !important;
|
| 1718 |
-
font-size: 0.85rem !important;
|
| 1719 |
-
}
|
| 1720 |
-
.pareto-scale-toggle label span {
|
| 1721 |
-
margin: 0 !important;
|
| 1722 |
-
padding: 0 !important;
|
| 1723 |
-
color: inherit !important;
|
| 1724 |
-
opacity: 1 !important;
|
| 1725 |
-
}
|
| 1726 |
-
.pareto-scale-toggle label > * + * {
|
| 1727 |
-
margin-left: 0 !important;
|
| 1728 |
-
}
|
| 1729 |
-
.pareto-scale-toggle label + label::before,
|
| 1730 |
-
.pareto-scale-toggle label + label {
|
| 1731 |
-
content: none !important;
|
| 1732 |
-
border-left: none !important;
|
| 1733 |
-
}
|
| 1734 |
-
.pareto-scale-toggle input[type="radio"] {
|
| 1735 |
-
position: absolute !important;
|
| 1736 |
-
appearance: none !important;
|
| 1737 |
-
opacity: 0 !important;
|
| 1738 |
-
width: 0 !important;
|
| 1739 |
-
height: 0 !important;
|
| 1740 |
-
margin: 0 !important;
|
| 1741 |
-
pointer-events: none !important;
|
| 1742 |
-
}
|
| 1743 |
-
.pareto-scale-toggle label:hover {
|
| 1744 |
-
background: transparent !important;
|
| 1745 |
-
color: var(--pruna-text-primary) !important;
|
| 1746 |
-
}
|
| 1747 |
-
.pareto-scale-toggle label.selected,
|
| 1748 |
-
.pareto-scale-toggle label:has(input:checked) {
|
| 1749 |
-
background: var(--pruna-toggle-thumb) !important;
|
| 1750 |
-
color: var(--pruna-lavender) !important;
|
| 1751 |
-
font-weight: 700 !important;
|
| 1752 |
-
border-color: transparent !important;
|
| 1753 |
-
box-shadow: var(--pruna-toggle-thumb-shadow) !important;
|
| 1754 |
-
}
|
| 1755 |
.pareto-layout,
|
| 1756 |
.pareto-layout.row,
|
| 1757 |
.pareto-layout .form {
|
|
@@ -1809,7 +1549,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
|
|
| 1809 |
.app-header .app-header-brand h1 {
|
| 1810 |
display: block !important;
|
| 1811 |
width: max-content !important;
|
| 1812 |
-
max-width:
|
| 1813 |
flex: 0 0 auto !important;
|
| 1814 |
margin: 0 !important;
|
| 1815 |
padding: 0 !important;
|
|
@@ -2286,7 +2026,7 @@ def load_qwen_combined_dataframe(path):
|
|
| 2286 |
df = df[~df["Model"].astype(str).str.startswith("#")].copy()
|
| 2287 |
df["Model"] = df["Model"].astype(str).str.strip()
|
| 2288 |
|
| 2289 |
-
|
| 2290 |
df,
|
| 2291 |
[
|
| 2292 |
"Price / Image (USD)",
|
|
@@ -2296,9 +2036,7 @@ def load_qwen_combined_dataframe(path):
|
|
| 2296 |
"Rapidata Elo",
|
| 2297 |
"Datapoint Elo",
|
| 2298 |
],
|
| 2299 |
-
)
|
| 2300 |
-
df = df.drop(columns=["Raw Win Rate"], errors="ignore")
|
| 2301 |
-
return df.reset_index(drop=True)
|
| 2302 |
|
| 2303 |
|
| 2304 |
df = load_oneig_dataframe(oneig_path)
|
|
@@ -2349,8 +2087,8 @@ qwen_combined_dir = _resolve_data_path(
|
|
| 2349 |
space_root.parent / "qwen_image_bench_combined",
|
| 2350 |
)
|
| 2351 |
qwen_path = _resolve_data_path(
|
| 2352 |
-
data_dir / "
|
| 2353 |
-
space_root.parent / "
|
| 2354 |
)
|
| 2355 |
aa_path = _resolve_data_path(
|
| 2356 |
data_dir / "artificial_analysis_text_to_image_leaderboard.csv",
|
|
@@ -2371,6 +2109,7 @@ qwen_display_columns = [
|
|
| 2371 |
"Datapoint Elo",
|
| 2372 |
"Rapidata Elo",
|
| 2373 |
"P-Judge Overall",
|
|
|
|
| 2374 |
"Median Generation Time (s)",
|
| 2375 |
"Min Generation Time (s)",
|
| 2376 |
"Price / Image (USD)",
|
|
@@ -2578,23 +2317,11 @@ custom_head = """
|
|
| 2578 |
return found;
|
| 2579 |
};
|
| 2580 |
|
| 2581 |
-
const applyPlotTheme = (gd, mode) => {
|
| 2582 |
-
const layout = PLOT_LAYOUT[mode];
|
| 2583 |
-
if (!layout || typeof Plotly === "undefined" || !gd) return;
|
| 2584 |
-
if (gd.layout && gd.layout.paper_bgcolor === layout.paper_bgcolor) return;
|
| 2585 |
-
try { Plotly.relayout(gd, layout); } catch (e) {}
|
| 2586 |
-
};
|
| 2587 |
-
|
| 2588 |
-
const watchPlotTheme = (gd) => {
|
| 2589 |
-
if (!gd || gd.__inferbenchThemeBound) return;
|
| 2590 |
-
gd.__inferbenchThemeBound = true;
|
| 2591 |
-
gd.addEventListener("plotly_afterplot", () => applyPlotTheme(gd, currentMode()));
|
| 2592 |
-
};
|
| 2593 |
-
|
| 2594 |
const restylePlots = (mode) => {
|
|
|
|
|
|
|
| 2595 |
queryAll(".js-plotly-plot").forEach((gd) => {
|
| 2596 |
-
|
| 2597 |
-
applyPlotTheme(gd, mode);
|
| 2598 |
});
|
| 2599 |
};
|
| 2600 |
|
|
|
|
| 62 |
--pruna-accordion-bg: rgba(255, 255, 255, 0.02);
|
| 63 |
--pruna-accordion-border: rgba(216, 180, 254, 0.15);
|
| 64 |
--pruna-dropdown-hover: #2a1844;
|
|
|
|
|
|
|
|
|
|
| 65 |
color-scheme: dark;
|
| 66 |
}
|
| 67 |
|
|
|
|
| 105 |
--pruna-accordion-bg: var(--pruna-bg-card);
|
| 106 |
--pruna-accordion-border: var(--pruna-border);
|
| 107 |
--pruna-dropdown-hover: #f3e8ff;
|
|
|
|
|
|
|
|
|
|
| 108 |
color-scheme: light;
|
| 109 |
}
|
| 110 |
|
| 111 |
+
html, body {
|
| 112 |
width: 100% !important;
|
| 113 |
max-width: 100% !important;
|
| 114 |
+
overflow-x: clip;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
-webkit-tap-highlight-color: transparent;
|
| 116 |
}
|
| 117 |
html, body, .gradio-container, .main {
|
|
|
|
| 134 |
/* Subtle depth — not a marketing-site hero glow */
|
| 135 |
body, .gradio-container {
|
| 136 |
background-image: var(--pruna-glow) !important;
|
| 137 |
+
background-attachment: fixed !important;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
}
|
| 139 |
|
| 140 |
.gradio-container {
|
| 141 |
width: 100% !important;
|
| 142 |
max-width: 1200px !important;
|
|
|
|
| 143 |
margin: 0 auto !important;
|
| 144 |
padding-top: 0 !important;
|
| 145 |
padding-left: 20px !important;
|
| 146 |
padding-right: 20px !important;
|
| 147 |
box-sizing: border-box !important;
|
| 148 |
+
overflow-x: clip;
|
| 149 |
}
|
| 150 |
.gradio-container .main,
|
| 151 |
.gradio-container .wrap,
|
|
|
|
| 166 |
.workspace-filters,
|
| 167 |
.view-filters {
|
| 168 |
max-width: 100% !important;
|
|
|
|
| 169 |
}
|
| 170 |
|
| 171 |
/* —— App header (P-Bench only) —— */
|
|
|
|
| 300 |
background: transparent !important;
|
| 301 |
box-shadow: none !important;
|
| 302 |
}
|
|
|
|
|
|
|
|
|
|
| 303 |
.workspace-shell > .tabs,
|
| 304 |
.workspace-shell > .main-tabs,
|
| 305 |
.workspace-shell > .block:not(.workspace-filters),
|
|
|
|
| 323 |
margin: 0 0 16px !important;
|
| 324 |
justify-content: center !important;
|
| 325 |
width: 100% !important;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 326 |
}
|
| 327 |
.main-tabs .tab-container {
|
| 328 |
height: auto !important;
|
|
|
|
| 329 |
justify-content: center !important;
|
| 330 |
flex-wrap: wrap !important;
|
| 331 |
+
overflow: hidden !important;
|
| 332 |
max-width: 100% !important;
|
| 333 |
gap: 2px;
|
| 334 |
}
|
|
|
|
| 445 |
.app-header-brand h1,
|
| 446 |
.gradio-container .app-header-brand h1 {
|
| 447 |
font-size: 1.55rem !important;
|
|
|
|
|
|
|
| 448 |
}
|
| 449 |
.app-header-tagline {
|
| 450 |
font-size: 0.88rem !important;
|
|
|
|
| 521 |
left: 0 !important;
|
| 522 |
width: 2.4rem;
|
| 523 |
min-width: 2.4rem;
|
|
|
|
| 524 |
}
|
| 525 |
.ranking-table .model-cell,
|
| 526 |
+
.prose .ranking-table .model-cell,
|
| 527 |
+
.ranking-table th.model-cell {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 528 |
position: sticky !important;
|
| 529 |
+
left: 2.4rem !important;
|
| 530 |
+
min-width: 108px;
|
| 531 |
+
max-width: 36vw;
|
|
|
|
|
|
|
|
|
|
| 532 |
}
|
| 533 |
+
.ranking-table .model-cell strong {
|
| 534 |
+
white-space: nowrap;
|
| 535 |
}
|
| 536 |
.compare-controls,
|
| 537 |
.compare-controls.row,
|
|
|
|
| 604 |
.view-filters {
|
| 605 |
display: flex !important;
|
| 606 |
flex-wrap: wrap !important;
|
| 607 |
+
align-items: end !important;
|
| 608 |
gap: 12px !important;
|
| 609 |
margin: 0;
|
| 610 |
overflow: visible !important;
|
|
|
|
| 612 |
.view-filters > div,
|
| 613 |
.view-filters > .block,
|
| 614 |
.view-filters > .form {
|
| 615 |
+
flex: 1 1 0 !important;
|
| 616 |
min-width: 0 !important;
|
| 617 |
}
|
| 618 |
.view-filters > .block,
|
|
|
|
| 764 |
line-height: 1.45 !important;
|
| 765 |
font-weight: 400 !important;
|
| 766 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 767 |
.view-filters span[data-testid="block-info"],
|
| 768 |
.view-filters .info,
|
| 769 |
.view-filters .block-info {
|
|
|
|
| 901 |
.leaderboard-controls {
|
| 902 |
display: flex !important;
|
| 903 |
flex-wrap: wrap !important;
|
| 904 |
+
align-items: end !important;
|
| 905 |
gap: 10px !important;
|
| 906 |
margin-bottom: 12px;
|
| 907 |
overflow: visible !important;
|
|
|
|
| 1317 |
box-shadow: none !important;
|
| 1318 |
}
|
| 1319 |
.compare-controls .compare-prompt-count .head {
|
| 1320 |
+
display: contents;
|
|
|
|
|
|
|
| 1321 |
margin: 0 !important;
|
| 1322 |
}
|
| 1323 |
.compare-controls .compare-prompt-count .head label {
|
|
|
|
| 1491 |
margin: 0 !important;
|
| 1492 |
}
|
| 1493 |
.compare-row { display: grid; gap: 12px; min-width: 0; width: 100%; }
|
| 1494 |
+
.compare-prompt-text { overflow-wrap: anywhere; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1495 |
.pareto-layout,
|
| 1496 |
.pareto-layout.row,
|
| 1497 |
.pareto-layout .form {
|
|
|
|
| 1549 |
.app-header .app-header-brand h1 {
|
| 1550 |
display: block !important;
|
| 1551 |
width: max-content !important;
|
| 1552 |
+
max-width: none !important;
|
| 1553 |
flex: 0 0 auto !important;
|
| 1554 |
margin: 0 !important;
|
| 1555 |
padding: 0 !important;
|
|
|
|
| 2026 |
df = df[~df["Model"].astype(str).str.startswith("#")].copy()
|
| 2027 |
df["Model"] = df["Model"].astype(str).str.strip()
|
| 2028 |
|
| 2029 |
+
return _as_numeric(
|
| 2030 |
df,
|
| 2031 |
[
|
| 2032 |
"Price / Image (USD)",
|
|
|
|
| 2036 |
"Rapidata Elo",
|
| 2037 |
"Datapoint Elo",
|
| 2038 |
],
|
| 2039 |
+
).reset_index(drop=True)
|
|
|
|
|
|
|
| 2040 |
|
| 2041 |
|
| 2042 |
df = load_oneig_dataframe(oneig_path)
|
|
|
|
| 2087 |
space_root.parent / "qwen_image_bench_combined",
|
| 2088 |
)
|
| 2089 |
qwen_path = _resolve_data_path(
|
| 2090 |
+
data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
|
| 2091 |
+
space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
|
| 2092 |
)
|
| 2093 |
aa_path = _resolve_data_path(
|
| 2094 |
data_dir / "artificial_analysis_text_to_image_leaderboard.csv",
|
|
|
|
| 2109 |
"Datapoint Elo",
|
| 2110 |
"Rapidata Elo",
|
| 2111 |
"P-Judge Overall",
|
| 2112 |
+
"Raw Win Rate",
|
| 2113 |
"Median Generation Time (s)",
|
| 2114 |
"Min Generation Time (s)",
|
| 2115 |
"Price / Image (USD)",
|
|
|
|
| 2317 |
return found;
|
| 2318 |
};
|
| 2319 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2320 |
const restylePlots = (mode) => {
|
| 2321 |
+
const layout = PLOT_LAYOUT[mode];
|
| 2322 |
+
if (!layout || typeof Plotly === "undefined") return;
|
| 2323 |
queryAll(".js-plotly-plot").forEach((gd) => {
|
| 2324 |
+
try { Plotly.relayout(gd, layout); } catch (e) {}
|
|
|
|
| 2325 |
});
|
| 2326 |
};
|
| 2327 |
|
data/qwen_image_bench_model_price_and_median_generation_time.csv
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Model,Price / Image (USD),Median Generation Time (s),Min Generation Time (s),P-Judge Overall,Raw Win Rate,Rapidata Elo,Datapoint Elo,Benchmark.ai Elo
|
| 2 |
+
reve_2_1,N/A,N/A,28.3,,,,,1173.2
|
| 3 |
+
ideogram_4_0_quality,N/A,N/A,66.6,,,,,1131.0
|
| 4 |
+
gpt_image_2,0.21,77.8,77.8,59.02083099999998,,1172.58,1116,1124.5
|
| 5 |
+
nano_banana_2_0,N/A,N/A,N/A,56.362956,59.4%,1071.79,1067,1056.3
|
| 6 |
+
gpt_image_1_5,0.135,38.0,38.0,57.864434999999986,64.5%,1102.09,1064,910.3
|
| 7 |
+
hidream_i1_dev,0.0086,2.823569217998738,2.06,49.17682099999999,51.7%,999.17,984,
|
| 8 |
+
gpt_image_1,0.167,38.8,38.8,54.975741000000006,,1095.39,987,
|
| 9 |
+
imagen_4_0_ultra,0.06,11.4,11.4,53.385603999999965,59.7%,1075.22,1023,
|
| 10 |
+
flux_2_flex,0.06,10.856533817990567,8.07,53.9245676767677,57.8%,1054.34,1018,921.3
|
| 11 |
+
qwen_image,0.025,4.8,4.8,51.561746,51.5%,1073.72,1005,975.6
|
| 12 |
+
seedream_5_0,N/A,N/A,N/A,55.715153,,1070.35,1012,
|
| 13 |
+
nano_banana_pro,0.134,17.2,17.2,56.452189898989914,58.0%,1029.51,1045,1102.4
|
| 14 |
+
hidream_i1_fast,0.0051,9.920469530501578,1.40,48.89298600000002,50.1%,1024.67,984,
|
| 15 |
+
imagen_4_fast,0.02,3.7531301500021073,2.71,50.155055208333344,,972.6,981,
|
| 16 |
+
seedream_4_5,0.04,16.6,16.6,55.66328800000001,,1048.96,1035,964.2
|
| 17 |
+
seedream_4_0,0.03,12.2,12.2,55.241443999999994,,1050.41,1035,
|
| 18 |
+
p_image_2_ideogram_low_1k,0.0075,2.59,1.55,54.827397,53.2%,1000.07,1009,1103.9
|
| 19 |
+
#p_image_2_ideogram_low_2k,0.016,5.17,4.25,53.971723,48.4%,1074.91,1003,
|
| 20 |
+
qwen_image_2_0_pro,0.035,35.5,35.5,56.181776,,1033.25,1017,959.9
|
| 21 |
+
juggernaut_base_flux,0.035,4.115834823496698,3.84,49.474676,46.4%,1038.22,972,
|
| 22 |
+
flux_2_pro,N/A,N/A,N/A,54.43423900000002,,993.54,1019,1011.6
|
| 23 |
+
z_image,0.005,1.5122045120006078,1.22,49.946227999999984,48.1%,1028.1,1001,
|
| 24 |
+
p_image_2_ideogram_high_1k,0.015,4.28,3.07,55.754507999999994,52.1%,972.92,1022,1104.0
|
| 25 |
+
#p_image_2_ideogram_high_2k,0.03,8.69,7.12,54.757842000000004,49.4%,1074.91,1007,
|
| 26 |
+
qwen_image_2512,0.02,19.1,19.1,51.677326,,1029.6,1009,
|
| 27 |
+
flux_2_max,0.07,26.4,26.4,54.047976,60.9%,955.19,1028,1001.6
|
| 28 |
+
flux_1_1_pro_ultra,0.06,9.026992494000297,6.20,50.358445,52.0%,979.56,995,
|
| 29 |
+
juggernaut_pro_flux,0.055,3.699636150500737,3.36,49.741183,46.4%,972.14,964,
|
| 30 |
+
flux_dev,0.025,1.6931055715031107,1.49,48.241479,42.1%,925.04,940,
|
| 31 |
+
p_image_2_ideogram_very_low_1k,0.003,2.56,1.49,53.51894200000001,48.1%,959.86,995,1071.7
|
| 32 |
+
#p_image_2_ideogram_very_low_2k,0.006,3.94,3.05,53.68248699999998,47.2%,962.38,980,
|
| 33 |
+
#p_image_2_ideogram_very_low_1k_no_upsampling,0.005,0.82,,46.33,,,,1071.7
|
| 34 |
+
#p_image_2_ideogram_very_low_2k_no_upsampling,0.005,2.18,,46.39,,,,1071.7
|
| 35 |
+
#p_image_2_ideogram_low_1k_no_upsampling,0.01,3.33,,45.32,,,,1103.9
|
| 36 |
+
#p_image_2_ideogram_low_2k_no_upsampling,0.01,3.34,,46.25,,,,1103.9
|
| 37 |
+
#p_image_2_ideogram_medium_1k_no_upsampling,0.015,2.13,,46.88,,,,1115.1
|
| 38 |
+
#p_image_2_ideogram_medium_2k_no_upsampling,0.015,5.55,,47.11,,,,1115.1
|
| 39 |
+
#p_image_2_ideogram_high_1k_no_upsampling,0.03,3.11,,46.88,,,,1104.0
|
| 40 |
+
#p_image_2_ideogram_high_2k_no_upsampling,0.03,5.55,,47.06,,,,1104.0
|
| 41 |
+
hidream_i1_full,0.014,6.008430051002506,5.69,46.82559300000002,36.9%,955.46,944,
|
| 42 |
+
flux_2_dev,0.025,4.310278721997747,4.03,52.71644489795918,53.1%,1007.6,1021,942.0
|
| 43 |
+
imagen_4_0,0.04,14.1,14.1,52.08996199999999,53.5%,979.62,1005,
|
| 44 |
+
wan_2_2_image,0.02,3.005390542501118,2.96,48.19959399999999,,944.87,960,
|
| 45 |
+
flux_krea,0.025,1.7150160090022837,1.7150160090022837,50.35734,50.0%,919.73,975,
|
| 46 |
+
p_image_2_ideogram_medium_1k,0.01,3.06,2.05,54.719193000000004,51.7%,941.46,1002,1115.1
|
| 47 |
+
#p_image_2_ideogram_medium_2k,0.02,7.44,6.47,54.23124444444446,48.1%,949.35,1000,
|
| 48 |
+
glm_image,0.05,188.2,188.2,51.42623399999999,,923.35,953,
|
| 49 |
+
p_image,0.005,1.0640762715011078,0.95,48.75217099999999,44.8%,924.37,961,1098.7
|
| 50 |
+
hunyuanimage_3_0,0.09,41.0,41.0,52.32440099999998,52.4%,1009.61,979,765.3
|
| 51 |
+
juggernaut_lightning_flux,0.006,1.1787893719956628,0.93,48.30471699999998,40.5%,916.68,929,
|
| 52 |
+
flux_1_1_pro,0.04,3.0104645500032348,2.34,49.92882700000001,50.6%,925.04,984,
|
| 53 |
+
flux_schnell,0.003,0.8411653029907029,0.80,46.685981818181816,34.8%,892.32,915,
|
| 54 |
+
kling_v2_1,N/A,N/A,N/A,51.044512,,870.04,981,
|
| 55 |
+
#p_image_2_ideogram_very_high_high_1k,0.075,9.34,7.25,58.45,,,1025,
|
| 56 |
+
#p_image_2_ideogram_very_high_low_1k,0.0375,26.17,10.44,57.59,,,1024,
|
| 57 |
+
#p_image_2_ideogram_very_high_medium_1k,0.05,9.28,5.96,57.88,,,1020,
|
| 58 |
+
#p_image_2_ideogram_very_high_very_low_1k,0.015,26.52,10.60,56.92,,,1002,
|
| 59 |
+
#p_image_2_ideogram_final_1k,0.0375,11.17,5.55,58.28,,,,
|
| 60 |
+
#p_image_2_ideogram_final_2k,0.075,14.34,9.96,57.68,,,,
|
data/qwen_image_bench_model_price_and_median_generation_time_10_august.csv
DELETED
|
@@ -1,64 +0,0 @@
|
|
| 1 |
-
Rapidata Model,Price / Image (USD),Median Generation Time (s),Min Generation Time (s),P-Judge Overall,Raw Win Rate,Rapidata Elo,Datapoint Elo,Benchmark.ai Elo
|
| 2 |
-
reve_2_1,N/A,N/A,28.3,,,,,1173.2
|
| 3 |
-
ideogram_4_0_quality,N/A,N/A,66.6,,,,,1131.0
|
| 4 |
-
gpt_image_2,0.21,77.8,77.8,59.02083099999998,65.6%,1172.58,1110,1124.5
|
| 5 |
-
nano_banana_2_0,N/A,N/A,N/A,56.362956,59.2%,1071.79,1063,1056.3
|
| 6 |
-
gpt_image_1_5,0.135,38.0,38.0,57.864434999999986,58.9%,1102.09,1060,910.3
|
| 7 |
-
hidream_i1_dev,0.0086,2.823569217998738,2.06,49.17682099999999,47.7%,999.17,983,
|
| 8 |
-
gpt_image_1,0.167,38.8,38.8,54.975741000000006,47.4%,1095.39,984,
|
| 9 |
-
imagen_4_0_ultra,0.06,11.4,11.4,53.385603999999965,52.9%,1075.22,1019,
|
| 10 |
-
flux_2_flex,0.06,10.856533817990567,8.07,53.9245676767677,52.4%,1054.34,1014,921.3
|
| 11 |
-
#flux_2_turbo,0.008,2.14,1.81,53.02,,,1003,
|
| 12 |
-
#flux_2_flash,0.005,1.43,1.03,52.03,,,1001,
|
| 13 |
-
qwen_image,0.025,4.8,4.8,51.561746,50.3%,1073.72,1002,975.6
|
| 14 |
-
seedream_5_0,N/A,N/A,N/A,55.715153,51.6%,1070.35,1010,
|
| 15 |
-
nano_banana_pro,0.134,17.2,17.2,56.452189898989914,56.4%,1029.51,1041,1102.4
|
| 16 |
-
hidream_i1_fast,0.0051,9.920469530501578,1.40,48.89298600000002,47.2%,1024.67,980,
|
| 17 |
-
imagen_4_fast,0.02,3.7531301500021073,2.71,50.155055208333344,47.2%,972.6,980,
|
| 18 |
-
seedream_4_5,0.04,16.6,16.6,55.66328800000001,54.7%,1048.96,1032,964.2
|
| 19 |
-
seedream_4_0,0.03,12.2,12.2,55.241443999999994,54.9%,1050.41,1032,
|
| 20 |
-
p_image_2_ideogram_low_1k,0.0075,2.59,1.55,54.827397,50.8%,1000.07,1006,1103.9
|
| 21 |
-
#p_image_2_ideogram_low_2k,0.016,5.17,4.25,53.971723,50.0%,1074.91,1000,1103.9
|
| 22 |
-
qwen_image_2_0_pro,0.035,35.5,35.5,56.181776,52.2%,1033.25,1014,959.9
|
| 23 |
-
juggernaut_base_flux,0.035,4.115834823496698,3.84,49.474676,46.1%,1038.22,971,
|
| 24 |
-
flux_2_pro,N/A,N/A,N/A,54.43423900000002,52.2%,993.54,1014,1011.6
|
| 25 |
-
z_image,0.005,1.5122045120006078,1.22,49.946227999999984,49.9%,1028.1,999,
|
| 26 |
-
p_image_2_ideogram_high_1k,0.015,4.28,3.07,55.754507999999994,52.5%,972.92,1017,1104.0
|
| 27 |
-
#p_image_2_ideogram_high_2k,0.06,8.69,7.12,54.757842000000004,50.7%,1074.91,1006,1104.0
|
| 28 |
-
qwen_image_2512,0.02,19.1,19.1,51.677326,50.9%,1029.6,1005,
|
| 29 |
-
flux_2_max,0.07,26.4,26.4,54.047976,53.9%,955.19,1026,1001.6
|
| 30 |
-
flux_1_1_pro_ultra,0.06,9.026992494000297,6.20,50.358445,48.6%,979.56,990,
|
| 31 |
-
juggernaut_pro_flux,0.055,3.699636150500737,3.36,49.741183,44.7%,972.14,963,
|
| 32 |
-
flux_dev,0.025,1.6931055715031107,1.49,48.241479,41.5%,925.04,941,
|
| 33 |
-
p_image_2_ideogram_very_low_1k,0.003,2.56,1.49,53.51894200000001,49.1%,959.86,994,1071.7
|
| 34 |
-
#p_image_2_ideogram_very_low_2k,0.006,3.94,3.05,53.68248699999998,46.3%,962.38,976,1071.7
|
| 35 |
-
#p_image_2_ideogram_very_low_1k_no_upsampling,0.005,0.82,,46.33,,,,1071.7
|
| 36 |
-
#p_image_2_ideogram_very_low_2k_no_upsampling,0.005,2.18,,46.39,,,,1071.7
|
| 37 |
-
#p_image_2_ideogram_low_1k_no_upsampling,0.01,3.33,,45.32,,,,1103.9
|
| 38 |
-
#p_image_2_ideogram_low_2k_no_upsampling,0.01,3.34,,46.25,,,,1103.9
|
| 39 |
-
#p_image_2_ideogram_medium_1k_no_upsampling,0.015,2.13,,46.88,,,,1115.1
|
| 40 |
-
#p_image_2_ideogram_medium_2k_no_upsampling,0.015,5.55,,47.11,,,,1115.1
|
| 41 |
-
#p_image_2_ideogram_high_1k_no_upsampling,0.03,3.11,,46.88,,,,1104.0
|
| 42 |
-
#p_image_2_ideogram_high_2k_no_upsampling,0.03,5.55,,47.06,,,,1104.0
|
| 43 |
-
hidream_i1_full,0.014,6.008430051002506,5.69,46.82559300000002,41.4%,955.46,941,
|
| 44 |
-
flux_2_dev,0.025,4.310278721997747,4.03,52.71644489795918,52.5%,1007.6,1016,942.0
|
| 45 |
-
imagen_4_0,0.04,14.1,14.1,52.08996199999999,50.4%,979.62,1002,
|
| 46 |
-
wan_2_2_image,0.02,3.005390542501118,2.96,48.19959399999999,44.1%,944.87,958,
|
| 47 |
-
flux_krea,0.025,1.7150160090022837,1.7150160090022837,50.35734,46.2%,919.73,973,
|
| 48 |
-
p_image_2_ideogram_medium_1k,0.01,3.06,2.05,54.719193000000004,49.8%,941.46,999,1115.1
|
| 49 |
-
#p_image_2_ideogram_medium_2k,0.02,7.44,6.47,54.23124444444446,49.3%,949.35,996,1115.1
|
| 50 |
-
glm_image,0.05,188.2,188.2,51.42623399999999,42.9%,923.35,950,
|
| 51 |
-
p_image,0.005,1.0640762715011078,0.95,48.75217099999999,44.3%,924.37,960,1098.7
|
| 52 |
-
hunyuanimage_3_0,0.09,41.0,41.0,52.32440099999998,46.6%,1009.61,977,765.3
|
| 53 |
-
juggernaut_lightning_flux,0.006,1.1787893719956628,0.93,48.30471699999998,40.0%,916.68,930,
|
| 54 |
-
flux_1_1_pro,0.04,3.0104645500032348,2.34,49.92882700000001,47.7%,925.04,984,
|
| 55 |
-
flux_schnell,0.003,0.8411653029907029,0.80,46.685981818181816,37.6%,892.32,914,
|
| 56 |
-
kling_v2_1,N/A,N/A,N/A,51.044512,47.2%,870.04,980,
|
| 57 |
-
#p_image_2_ideogram_very_high_high_1k,0.075,9.34,7.25,58.45,52.9%,,1024,
|
| 58 |
-
#p_image_2_ideogram_very_high_low_1k,0.0375,26.17,10.44,57.59,52.9%,,1023,
|
| 59 |
-
#p_image_2_ideogram_very_high_medium_1k,0.05,9.28,5.96,57.88,52.3%,,1019,
|
| 60 |
-
#p_image_2_ideogram_very_high_very_low_1k,0.015,26.52,10.60,56.92,49.4%,,1001,
|
| 61 |
-
p_image_2_ideogram_very_high_1k,0.033,11.17,5.55,58.28,55.5%,,1034,
|
| 62 |
-
#p_image_2_ideogram_very_high_2k,0.066,14.34,9.96,57.68,53.4%,,1020,
|
| 63 |
-
#p_image_2_ideogram_very_high_john_1k,,8.53,6.43,58.19,,,1015,
|
| 64 |
-
#p_image_2_ideogram_very_high_john_2k,,13.86,10.64,57.75,,,1002,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
model_display.py
CHANGED
|
@@ -87,7 +87,6 @@ MODEL_DISPLAY_NAMES = {
|
|
| 87 |
"p_image_2_ideogram_high_1k": "P-Image-Ideogram High 1K",
|
| 88 |
"p_image_2_ideogram_high_2k": "P-Image-Ideogram High 2K",
|
| 89 |
"P-Image-Ideogram (High)": "P-Image-Ideogram High",
|
| 90 |
-
"p_image_2_ideogram_very_high_1k": "P-Image-Ideogram Very High 1K",
|
| 91 |
# Others overlapping P-Bench
|
| 92 |
"z_image": "Z-Image",
|
| 93 |
"glm_image": "GLM-Image",
|
|
|
|
| 87 |
"p_image_2_ideogram_high_1k": "P-Image-Ideogram High 1K",
|
| 88 |
"p_image_2_ideogram_high_2k": "P-Image-Ideogram High 2K",
|
| 89 |
"P-Image-Ideogram (High)": "P-Image-Ideogram High",
|
|
|
|
| 90 |
# Others overlapping P-Bench
|
| 91 |
"z_image": "Z-Image",
|
| 92 |
"glm_image": "GLM-Image",
|
ui.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
| 1 |
from html import escape
|
| 2 |
-
from math import ceil, floor, log10
|
| 3 |
from pathlib import Path
|
| 4 |
import base64
|
| 5 |
import random
|
|
@@ -26,12 +25,6 @@ MAX_PARETO_METRICS = 8
|
|
| 26 |
_PARETO_SLOT_COUNT = 1 + MAX_PARETO_METRICS * 8
|
| 27 |
_PARETO_PRICE_COLUMN = "Price / Image (USD)"
|
| 28 |
_PARETO_TIME_COLUMN = "Min Generation Time (s)"
|
| 29 |
-
_PARETO_SCALE_CHOICES = [
|
| 30 |
-
("Log", "Logarithmic"),
|
| 31 |
-
("Linear", "Linear"),
|
| 32 |
-
]
|
| 33 |
-
_PARETO_SCALE_VALUES = {value for _, value in _PARETO_SCALE_CHOICES}
|
| 34 |
-
_PARETO_SCALE_DEFAULT = "Logarithmic"
|
| 35 |
|
| 36 |
TAB_LEADERBOARDS = "leaderboards"
|
| 37 |
TAB_PARETO = "pareto"
|
|
@@ -251,14 +244,6 @@ def _dataset_has_samples(datasets, dataset_id):
|
|
| 251 |
return bool(dataset and dataset.get("samples"))
|
| 252 |
|
| 253 |
|
| 254 |
-
def _sample_model_ids(datasets, dataset_id):
|
| 255 |
-
dataset = _item(datasets, dataset_id)
|
| 256 |
-
samples = dataset.get("samples") if dataset else None
|
| 257 |
-
if not samples:
|
| 258 |
-
return set()
|
| 259 |
-
return set(samples.get("models") or [])
|
| 260 |
-
|
| 261 |
-
|
| 262 |
def _dataset_has_pareto(datasets, dataset_id):
|
| 263 |
dataset = _item(datasets, dataset_id)
|
| 264 |
columns = getattr(dataset.get("data") if dataset else None, "columns", [])
|
|
@@ -329,25 +314,23 @@ def _metric_dropdown_value(metric_id):
|
|
| 329 |
]
|
| 330 |
|
| 331 |
|
| 332 |
-
def _model_choices(datasets, dataset_id
|
| 333 |
cached = _MODEL_CHOICES_CACHE.get(dataset_id)
|
| 334 |
-
if cached is None:
|
| 335 |
-
dataset = _item(datasets, dataset_id)
|
| 336 |
-
data = dataset.get("data") if dataset else None
|
| 337 |
-
if data is None or "Model" not in getattr(data, "columns", []):
|
| 338 |
-
cached = []
|
| 339 |
-
else:
|
| 340 |
-
models = data["Model"].dropna().astype(str).unique().tolist()
|
| 341 |
-
# (label, value) so the UI shows the shared name but filters on the raw id.
|
| 342 |
-
cached = sorted(
|
| 343 |
-
((display_model_name(model), model) for model in models),
|
| 344 |
-
key=lambda item: item[0].casefold(),
|
| 345 |
-
)
|
| 346 |
-
_MODEL_CHOICES_CACHE[dataset_id] = cached
|
| 347 |
-
if not require_samples:
|
| 348 |
return cached
|
| 349 |
-
|
| 350 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 351 |
|
| 352 |
|
| 353 |
def _model_choice_values(choices):
|
|
@@ -602,6 +585,7 @@ def _display_label(column):
|
|
| 602 |
"Arena Art Elo": "Art",
|
| 603 |
"Arena Portraits Elo": "Portraits",
|
| 604 |
"Arena Text Rendering Elo": "Text Rendering",
|
|
|
|
| 605 |
"Median Generation Time (s)": "Median generation time",
|
| 606 |
"Min Generation Time (s)": "Min generation time",
|
| 607 |
"Price / Image (USD)": "Price per image",
|
|
@@ -686,7 +670,6 @@ def _build_pareto_figure(
|
|
| 686 |
x_title,
|
| 687 |
x_hover_prefix="",
|
| 688 |
x_hover_suffix="",
|
| 689 |
-
x_axis_type="linear",
|
| 690 |
):
|
| 691 |
scatter = (
|
| 692 |
data[["Model", score_column, x_column]]
|
|
@@ -772,29 +755,7 @@ def _build_pareto_figure(
|
|
| 772 |
)
|
| 773 |
axis_font = {"color": "#fafafa", "size": 13}
|
| 774 |
tick_font = {"color": "#a3a3a3", "size": 12}
|
| 775 |
-
x_axis_ticks = {}
|
| 776 |
-
if x_axis_type == "log":
|
| 777 |
-
positive_x = scatter.loc[scatter[x_column] > 0, x_column].astype(float)
|
| 778 |
-
if not positive_x.empty:
|
| 779 |
-
minimum = positive_x.min()
|
| 780 |
-
maximum = positive_x.max()
|
| 781 |
-
tick_values = [
|
| 782 |
-
factor * (10**exponent)
|
| 783 |
-
for exponent in range(
|
| 784 |
-
floor(log10(minimum)),
|
| 785 |
-
ceil(log10(maximum)) + 1,
|
| 786 |
-
)
|
| 787 |
-
for factor in (1, 2, 5)
|
| 788 |
-
if minimum * 0.8 <= factor * (10**exponent) <= maximum * 1.2
|
| 789 |
-
]
|
| 790 |
-
x_axis_ticks = {
|
| 791 |
-
"tickmode": "array",
|
| 792 |
-
"tickvals": tick_values,
|
| 793 |
-
"ticktext": [f"{value:g}" for value in tick_values],
|
| 794 |
-
}
|
| 795 |
fig.update_xaxes(
|
| 796 |
-
type=x_axis_type,
|
| 797 |
-
**x_axis_ticks,
|
| 798 |
showgrid=True,
|
| 799 |
gridcolor="rgba(74, 57, 98, 0.55)",
|
| 800 |
zeroline=False,
|
|
@@ -813,55 +774,6 @@ def _build_pareto_figure(
|
|
| 813 |
return fig
|
| 814 |
|
| 815 |
|
| 816 |
-
def _is_log_scale(scale):
|
| 817 |
-
return scale == "Logarithmic"
|
| 818 |
-
|
| 819 |
-
|
| 820 |
-
def _pareto_axis_type(scale):
|
| 821 |
-
return "log" if _is_log_scale(scale) else "linear"
|
| 822 |
-
|
| 823 |
-
|
| 824 |
-
def _pareto_scale_radio(*extra_classes):
|
| 825 |
-
return gr.Radio(
|
| 826 |
-
choices=_PARETO_SCALE_CHOICES,
|
| 827 |
-
value=_PARETO_SCALE_DEFAULT,
|
| 828 |
-
show_label=False,
|
| 829 |
-
container=False,
|
| 830 |
-
elem_classes=["pareto-scale-toggle", *extra_classes],
|
| 831 |
-
)
|
| 832 |
-
|
| 833 |
-
|
| 834 |
-
def _pareto_plot_heading(title):
|
| 835 |
-
with gr.Row(equal_height=False, elem_classes="pareto-heading-row"):
|
| 836 |
-
gr.Markdown(f"#### {title}", elem_classes="pareto-subhead")
|
| 837 |
-
with gr.Column(min_width=140, elem_classes="pareto-scale-control"):
|
| 838 |
-
return _pareto_scale_radio()
|
| 839 |
-
|
| 840 |
-
|
| 841 |
-
def _default_pareto_scales():
|
| 842 |
-
return [_PARETO_SCALE_DEFAULT] * MAX_PARETO_METRICS
|
| 843 |
-
|
| 844 |
-
|
| 845 |
-
def _normalize_pareto_scales(scales):
|
| 846 |
-
values = list(scales or [])
|
| 847 |
-
if len(values) < MAX_PARETO_METRICS:
|
| 848 |
-
values.extend(
|
| 849 |
-
[_PARETO_SCALE_DEFAULT] * (MAX_PARETO_METRICS - len(values))
|
| 850 |
-
)
|
| 851 |
-
return values[:MAX_PARETO_METRICS]
|
| 852 |
-
|
| 853 |
-
|
| 854 |
-
def _uniform_pareto_scales(scale):
|
| 855 |
-
return [scale] * MAX_PARETO_METRICS
|
| 856 |
-
|
| 857 |
-
|
| 858 |
-
def _pareto_master_scale_update(price_scales, time_scales):
|
| 859 |
-
values = list(price_scales) + list(time_scales)
|
| 860 |
-
if values and all(value == values[0] for value in values):
|
| 861 |
-
return gr.update(value=values[0])
|
| 862 |
-
return gr.update(value=None)
|
| 863 |
-
|
| 864 |
-
|
| 865 |
def _pareto_axis(data, score_column, x_column, x_title, missing_message, empty_message, **hover):
|
| 866 |
if x_column not in data.columns:
|
| 867 |
return None, missing_message
|
|
@@ -877,12 +789,7 @@ def _pareto_axis(data, score_column, x_column, x_title, missing_message, empty_m
|
|
| 877 |
return fig, None
|
| 878 |
|
| 879 |
|
| 880 |
-
def _pareto_pair(
|
| 881 |
-
data,
|
| 882 |
-
score_column,
|
| 883 |
-
latency_scale=_PARETO_SCALE_DEFAULT,
|
| 884 |
-
price_scale=_PARETO_SCALE_DEFAULT,
|
| 885 |
-
):
|
| 886 |
score_missing = "No score data is available for this metric."
|
| 887 |
if data is None or not score_column or score_column not in data.columns:
|
| 888 |
return None, score_missing, None, score_missing
|
|
@@ -895,7 +802,6 @@ def _pareto_pair(
|
|
| 895 |
"Price per image isn't available for this dataset.",
|
| 896 |
"No models have both a score and a price for this metric.",
|
| 897 |
x_hover_prefix="$",
|
| 898 |
-
x_axis_type=_pareto_axis_type(price_scale),
|
| 899 |
)
|
| 900 |
time_fig, time_message = _pareto_axis(
|
| 901 |
data,
|
|
@@ -905,7 +811,6 @@ def _pareto_pair(
|
|
| 905 |
"Min generation time isn't available for this dataset.",
|
| 906 |
"No models have both a score and a min generation time for this metric.",
|
| 907 |
x_hover_suffix="s",
|
| 908 |
-
x_axis_type=_pareto_axis_type(latency_scale),
|
| 909 |
)
|
| 910 |
return price_fig, price_message, time_fig, time_message
|
| 911 |
|
|
@@ -944,16 +849,9 @@ def _pareto_slot_note(price_fig, price_message, time_fig, time_message, data):
|
|
| 944 |
return " ".join(notes)
|
| 945 |
|
| 946 |
|
| 947 |
-
def _pareto_slot_updates(
|
| 948 |
-
data,
|
| 949 |
-
score_columns,
|
| 950 |
-
price_scales=None,
|
| 951 |
-
time_scales=None,
|
| 952 |
-
):
|
| 953 |
"""Updates for a fixed bank of Gradio Plot slots (visible/hidden)."""
|
| 954 |
score_columns = [column for column in (score_columns or []) if column]
|
| 955 |
-
price_scales = _normalize_pareto_scales(price_scales)
|
| 956 |
-
time_scales = _normalize_pareto_scales(time_scales)
|
| 957 |
has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
|
| 958 |
has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
|
| 959 |
dataset_note = _pareto_dataset_message(data)
|
|
@@ -975,10 +873,7 @@ def _pareto_slot_updates(
|
|
| 975 |
continue
|
| 976 |
score_column = score_columns[index]
|
| 977 |
price_fig, price_message, time_fig, time_message = _pareto_pair(
|
| 978 |
-
data,
|
| 979 |
-
score_column,
|
| 980 |
-
latency_scale=time_scales[index],
|
| 981 |
-
price_scale=price_scales[index],
|
| 982 |
)
|
| 983 |
show_price = price_fig is not None
|
| 984 |
show_time = time_fig is not None
|
|
@@ -1005,30 +900,6 @@ def _pareto_slot_updates(
|
|
| 1005 |
return updates
|
| 1006 |
|
| 1007 |
|
| 1008 |
-
def _pareto_all_scale_updates(data, score_columns, scale):
|
| 1009 |
-
"""Apply one scale to every Pareto plot and radio."""
|
| 1010 |
-
score_columns = [column for column in (score_columns or []) if column]
|
| 1011 |
-
price_updates = []
|
| 1012 |
-
time_updates = []
|
| 1013 |
-
for index in range(MAX_PARETO_METRICS):
|
| 1014 |
-
if index >= len(score_columns):
|
| 1015 |
-
price_updates.append(gr.skip())
|
| 1016 |
-
time_updates.append(gr.skip())
|
| 1017 |
-
continue
|
| 1018 |
-
price_fig, _, time_fig, _ = _pareto_pair(
|
| 1019 |
-
data,
|
| 1020 |
-
score_columns[index],
|
| 1021 |
-
latency_scale=scale,
|
| 1022 |
-
price_scale=scale,
|
| 1023 |
-
)
|
| 1024 |
-
price_updates.append(_pareto_plot_update(price_fig))
|
| 1025 |
-
time_updates.append(_pareto_plot_update(time_fig))
|
| 1026 |
-
radio_updates = [
|
| 1027 |
-
gr.update(value=scale) for _ in range(MAX_PARETO_METRICS * 2)
|
| 1028 |
-
]
|
| 1029 |
-
return price_updates + time_updates + radio_updates
|
| 1030 |
-
|
| 1031 |
-
|
| 1032 |
def _samples_html(samples, selected_models, num_prompts, seed=0):
|
| 1033 |
if not samples:
|
| 1034 |
return _pareto_unavailable_html(
|
|
@@ -1173,10 +1044,10 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1173 |
gr.Markdown(
|
| 1174 |
"<p class='filter-help'>"
|
| 1175 |
"These filters apply to Leaderboards, Pareto plots, and Samples. "
|
| 1176 |
-
"On Samples, only datasets
|
| 1177 |
-
"
|
| 1178 |
-
"
|
| 1179 |
-
"
|
| 1180 |
"</p>",
|
| 1181 |
elem_classes="filter-help-host",
|
| 1182 |
)
|
|
@@ -1246,27 +1117,12 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1246 |
) as pp_tab:
|
| 1247 |
gr.Markdown(
|
| 1248 |
"<p class='view-help'>"
|
| 1249 |
-
"Score against price and generation time. Green points are on "
|
| 1250 |
-
"
|
| 1251 |
-
"
|
| 1252 |
-
"</p>"
|
| 1253 |
-
"<p class='view-help'>"
|
| 1254 |
-
"Plots use a logarithmic scale by default. You can switch "
|
| 1255 |
-
"to linear for all plots, or individually for each plot."
|
| 1256 |
"</p>",
|
| 1257 |
elem_classes="view-help-host",
|
| 1258 |
)
|
| 1259 |
-
with gr.Row(
|
| 1260 |
-
equal_height=False,
|
| 1261 |
-
elem_classes="pareto-scale-all-row",
|
| 1262 |
-
):
|
| 1263 |
-
gr.HTML(
|
| 1264 |
-
"<span class='pareto-scale-all-label'>All plots</span>",
|
| 1265 |
-
padding=False,
|
| 1266 |
-
)
|
| 1267 |
-
pareto_all_scale = _pareto_scale_radio(
|
| 1268 |
-
"pareto-scale-toggle-all",
|
| 1269 |
-
)
|
| 1270 |
pareto_dataset_note = gr.HTML(
|
| 1271 |
"",
|
| 1272 |
padding=False,
|
|
@@ -1292,8 +1148,9 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1292 |
min_width=320,
|
| 1293 |
elem_classes="pareto-col",
|
| 1294 |
) as slot_price_col:
|
| 1295 |
-
|
| 1296 |
-
"Price vs score"
|
|
|
|
| 1297 |
)
|
| 1298 |
slot_price = gr.Plot(
|
| 1299 |
value=None,
|
|
@@ -1305,8 +1162,9 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1305 |
min_width=320,
|
| 1306 |
elem_classes="pareto-col",
|
| 1307 |
) as slot_time_col:
|
| 1308 |
-
|
| 1309 |
-
"Min generation time vs score"
|
|
|
|
| 1310 |
)
|
| 1311 |
slot_time = gr.Plot(
|
| 1312 |
value=None,
|
|
@@ -1327,10 +1185,8 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1327 |
slot_layout,
|
| 1328 |
slot_price_col,
|
| 1329 |
slot_price,
|
| 1330 |
-
slot_price_scale,
|
| 1331 |
slot_time_col,
|
| 1332 |
slot_time,
|
| 1333 |
-
slot_time_scale,
|
| 1334 |
)
|
| 1335 |
)
|
| 1336 |
|
|
@@ -1379,16 +1235,12 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1379 |
with gr.TabItem("About", id=TAB_ABOUT) as about_tab:
|
| 1380 |
render_about()
|
| 1381 |
|
| 1382 |
-
def _synced_filters(
|
| 1383 |
-
dataset_id, metric_id, models, *, clear_metric=False, require_samples=False
|
| 1384 |
-
):
|
| 1385 |
if clear_metric:
|
| 1386 |
metric_id = []
|
| 1387 |
else:
|
| 1388 |
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
| 1389 |
-
model_choices = _model_choices(
|
| 1390 |
-
datasets, dataset_id, require_samples=require_samples
|
| 1391 |
-
)
|
| 1392 |
model_values = set(_model_choice_values(model_choices))
|
| 1393 |
models = [model for model in (models or []) if model in model_values]
|
| 1394 |
metric_choices = _metric_dropdown_choices(datasets, metrics, dataset_id)
|
|
@@ -1472,8 +1324,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1472 |
"optimized": list(
|
| 1473 |
extras.get("optimized", prev.get("optimized") or [])
|
| 1474 |
),
|
| 1475 |
-
"price_scales": _normalize_pareto_scales(prev.get("price_scales")),
|
| 1476 |
-
"time_scales": _normalize_pareto_scales(prev.get("time_scales")),
|
| 1477 |
"stale": {
|
| 1478 |
TAB_LEADERBOARDS: not flags["include_leaderboard"],
|
| 1479 |
TAB_PARETO: not flags["include_pareto"],
|
|
@@ -1533,8 +1383,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1533 |
include_leaderboard=True,
|
| 1534 |
include_pareto=False,
|
| 1535 |
include_samples=False,
|
| 1536 |
-
price_scales=None,
|
| 1537 |
-
time_scales=None,
|
| 1538 |
):
|
| 1539 |
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 1540 |
data = view["data"]
|
|
@@ -1555,12 +1403,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1555 |
ranking_html = gr.skip()
|
| 1556 |
if include_pareto:
|
| 1557 |
pareto_data = _filter_leaderboard(data, [], [], [], models=models)
|
| 1558 |
-
pareto_updates = _pareto_slot_updates(
|
| 1559 |
-
pareto_data,
|
| 1560 |
-
view["score_columns"],
|
| 1561 |
-
price_scales=price_scales,
|
| 1562 |
-
time_scales=time_scales,
|
| 1563 |
-
)
|
| 1564 |
else:
|
| 1565 |
pareto_updates = _pareto_skip_updates()
|
| 1566 |
if include_samples:
|
|
@@ -1602,20 +1445,10 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1602 |
dataset_changed = source == "dataset" and dataset_id != view_state.get(
|
| 1603 |
"dataset_id"
|
| 1604 |
)
|
| 1605 |
-
|
| 1606 |
-
can_samples = _dataset_has_samples(datasets, dataset_id)
|
| 1607 |
-
selected_tab = tab
|
| 1608 |
if source == "dataset":
|
| 1609 |
-
if tab == TAB_SAMPLES and not can_samples:
|
| 1610 |
-
selected_tab = TAB_LEADERBOARDS
|
| 1611 |
-
elif tab == TAB_PARETO and not can_pareto:
|
| 1612 |
-
selected_tab = TAB_LEADERBOARDS
|
| 1613 |
synced = _synced_filters(
|
| 1614 |
-
dataset_id,
|
| 1615 |
-
metric_id,
|
| 1616 |
-
models,
|
| 1617 |
-
clear_metric=dataset_changed,
|
| 1618 |
-
require_samples=selected_tab == TAB_SAMPLES,
|
| 1619 |
)
|
| 1620 |
dataset_id, metric_id, models = synced[:3]
|
| 1621 |
metric_update, models_update = synced[3], synced[4]
|
|
@@ -1653,13 +1486,20 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1653 |
):
|
| 1654 |
return None
|
| 1655 |
|
|
|
|
| 1656 |
extras = (
|
| 1657 |
list(platform_value or []),
|
| 1658 |
list(owner_value or []),
|
| 1659 |
list(optimized_value or []),
|
| 1660 |
)
|
| 1661 |
extra_updates = None
|
|
|
|
|
|
|
| 1662 |
if source == "dataset":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1663 |
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 1664 |
extra_updates = _leaderboard_extras(
|
| 1665 |
view["data"] if view else None,
|
|
@@ -1697,8 +1537,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1697 |
extras[2],
|
| 1698 |
num_prompts,
|
| 1699 |
seed,
|
| 1700 |
-
price_scales=view_state.get("price_scales"),
|
| 1701 |
-
time_scales=view_state.get("time_scales"),
|
| 1702 |
**flags,
|
| 1703 |
),
|
| 1704 |
"state": _commit_state(
|
|
@@ -1839,17 +1677,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1839 |
)
|
| 1840 |
dataset_update = _dataset_dropdown_update(datasets, tab, dataset_id)
|
| 1841 |
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
| 1842 |
-
|
| 1843 |
-
model_choices = _model_choices(
|
| 1844 |
-
datasets, dataset_id, require_samples=require_samples
|
| 1845 |
-
)
|
| 1846 |
-
allowed_models = set(_model_choice_values(model_choices))
|
| 1847 |
-
models = [model for model in (models or []) if model in allowed_models]
|
| 1848 |
-
models_update = (
|
| 1849 |
-
gr.update(choices=model_choices, value=models)
|
| 1850 |
-
if (prev_tab == TAB_SAMPLES) != require_samples
|
| 1851 |
-
else gr.skip()
|
| 1852 |
-
)
|
| 1853 |
view_state["current_tab"] = tab
|
| 1854 |
view_state["dataset_id"] = dataset_id
|
| 1855 |
view_state["metric_id"] = metric_id
|
|
@@ -1886,7 +1714,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1886 |
filters_vis,
|
| 1887 |
dataset_update,
|
| 1888 |
metric_vis,
|
| 1889 |
-
|
| 1890 |
*lb_filters,
|
| 1891 |
)
|
| 1892 |
tab_select = (
|
|
@@ -1911,8 +1739,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1911 |
optimized_value,
|
| 1912 |
num_prompts,
|
| 1913 |
seed,
|
| 1914 |
-
price_scales=view_state.get("price_scales"),
|
| 1915 |
-
time_scales=view_state.get("time_scales"),
|
| 1916 |
**flags,
|
| 1917 |
)
|
| 1918 |
stale[tab] = False
|
|
@@ -1969,70 +1795,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1969 |
next_seed,
|
| 1970 |
)
|
| 1971 |
|
| 1972 |
-
def _on_pareto_plot_scale(slot_index, axis):
|
| 1973 |
-
def handler(dataset_id, metric_id, models, scale, view_state):
|
| 1974 |
-
view_state = dict(view_state or {})
|
| 1975 |
-
price_scales = _normalize_pareto_scales(
|
| 1976 |
-
view_state.get("price_scales")
|
| 1977 |
-
)
|
| 1978 |
-
time_scales = _normalize_pareto_scales(
|
| 1979 |
-
view_state.get("time_scales")
|
| 1980 |
-
)
|
| 1981 |
-
if axis == "price":
|
| 1982 |
-
if price_scales[slot_index] == scale:
|
| 1983 |
-
return gr.skip(), gr.skip(), gr.skip()
|
| 1984 |
-
price_scales[slot_index] = scale
|
| 1985 |
-
else:
|
| 1986 |
-
if time_scales[slot_index] == scale:
|
| 1987 |
-
return gr.skip(), gr.skip(), gr.skip()
|
| 1988 |
-
time_scales[slot_index] = scale
|
| 1989 |
-
view_state["price_scales"] = price_scales
|
| 1990 |
-
view_state["time_scales"] = time_scales
|
| 1991 |
-
master_scale = _pareto_master_scale_update(
|
| 1992 |
-
price_scales, time_scales
|
| 1993 |
-
)
|
| 1994 |
-
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 1995 |
-
score_columns = [
|
| 1996 |
-
column for column in (view["score_columns"] or []) if column
|
| 1997 |
-
]
|
| 1998 |
-
if slot_index >= len(score_columns):
|
| 1999 |
-
return gr.skip(), master_scale, view_state
|
| 2000 |
-
data = _filter_leaderboard(
|
| 2001 |
-
view["data"], [], [], [], models=list(models or [])
|
| 2002 |
-
)
|
| 2003 |
-
price_fig, _, time_fig, _ = _pareto_pair(
|
| 2004 |
-
data,
|
| 2005 |
-
score_columns[slot_index],
|
| 2006 |
-
latency_scale=time_scales[slot_index],
|
| 2007 |
-
price_scale=price_scales[slot_index],
|
| 2008 |
-
)
|
| 2009 |
-
fig = price_fig if axis == "price" else time_fig
|
| 2010 |
-
return _pareto_plot_update(fig), master_scale, view_state
|
| 2011 |
-
|
| 2012 |
-
handler.__name__ = f"on_pareto_{axis}_scale_{slot_index}"
|
| 2013 |
-
return handler
|
| 2014 |
-
|
| 2015 |
-
def on_pareto_all_scale(dataset_id, metric_id, models, scale, view_state):
|
| 2016 |
-
if scale not in _PARETO_SCALE_VALUES:
|
| 2017 |
-
return (*_skip_all(MAX_PARETO_METRICS * 4), gr.skip())
|
| 2018 |
-
view_state = dict(view_state or {})
|
| 2019 |
-
scales = _uniform_pareto_scales(scale)
|
| 2020 |
-
if (
|
| 2021 |
-
_normalize_pareto_scales(view_state.get("price_scales")) == scales
|
| 2022 |
-
and _normalize_pareto_scales(view_state.get("time_scales")) == scales
|
| 2023 |
-
):
|
| 2024 |
-
return (*_skip_all(MAX_PARETO_METRICS * 4), gr.skip())
|
| 2025 |
-
view_state["price_scales"] = scales
|
| 2026 |
-
view_state["time_scales"] = scales
|
| 2027 |
-
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 2028 |
-
data = _filter_leaderboard(
|
| 2029 |
-
view["data"], [], [], [], models=list(models or [])
|
| 2030 |
-
)
|
| 2031 |
-
return (
|
| 2032 |
-
*_pareto_all_scale_updates(data, view["score_columns"], scale),
|
| 2033 |
-
view_state,
|
| 2034 |
-
)
|
| 2035 |
-
|
| 2036 |
def _on_tab(tab):
|
| 2037 |
def handler(
|
| 2038 |
dataset_id,
|
|
@@ -2070,8 +1832,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 2070 |
"platform": [],
|
| 2071 |
"owner": [],
|
| 2072 |
"optimized": [],
|
| 2073 |
-
"price_scales": _default_pareto_scales(),
|
| 2074 |
-
"time_scales": _default_pareto_scales(),
|
| 2075 |
"stale": {
|
| 2076 |
TAB_LEADERBOARDS: False,
|
| 2077 |
TAB_PARETO: True,
|
|
@@ -2083,7 +1843,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 2083 |
pareto_dataset_note,
|
| 2084 |
*[
|
| 2085 |
component
|
| 2086 |
-
for slot_group, slot_title, slot_note, slot_layout, slot_price_col, slot_price,
|
| 2087 |
for component in (
|
| 2088 |
slot_group,
|
| 2089 |
slot_title,
|
|
@@ -2096,24 +1856,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 2096 |
)
|
| 2097 |
],
|
| 2098 |
]
|
| 2099 |
-
pareto_all_scale_outputs = [
|
| 2100 |
-
*[
|
| 2101 |
-
slot_price
|
| 2102 |
-
for _, _, _, _, _, slot_price, _, _, _, _ in pareto_slots
|
| 2103 |
-
],
|
| 2104 |
-
*[
|
| 2105 |
-
slot_time
|
| 2106 |
-
for _, _, _, _, _, _, _, _, slot_time, _ in pareto_slots
|
| 2107 |
-
],
|
| 2108 |
-
*[
|
| 2109 |
-
slot_price_scale
|
| 2110 |
-
for _, _, _, _, _, _, slot_price_scale, _, _, _ in pareto_slots
|
| 2111 |
-
],
|
| 2112 |
-
*[
|
| 2113 |
-
slot_time_scale
|
| 2114 |
-
for _, _, _, _, _, _, _, _, _, slot_time_scale in pareto_slots
|
| 2115 |
-
],
|
| 2116 |
-
]
|
| 2117 |
view_inputs = [
|
| 2118 |
platform,
|
| 2119 |
owner,
|
|
@@ -2218,56 +1960,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 2218 |
show_progress="hidden",
|
| 2219 |
)
|
| 2220 |
|
| 2221 |
-
pareto_all_scale.change(
|
| 2222 |
-
on_pareto_all_scale,
|
| 2223 |
-
inputs=[
|
| 2224 |
-
dataset_dd,
|
| 2225 |
-
metric_dd,
|
| 2226 |
-
models_dd,
|
| 2227 |
-
pareto_all_scale,
|
| 2228 |
-
view_state,
|
| 2229 |
-
],
|
| 2230 |
-
outputs=[*pareto_all_scale_outputs, view_state],
|
| 2231 |
-
**_VIEW_EVENTS,
|
| 2232 |
-
)
|
| 2233 |
-
|
| 2234 |
-
for slot_index, (
|
| 2235 |
-
_,
|
| 2236 |
-
_,
|
| 2237 |
-
_,
|
| 2238 |
-
_,
|
| 2239 |
-
_,
|
| 2240 |
-
slot_price,
|
| 2241 |
-
slot_price_scale,
|
| 2242 |
-
_,
|
| 2243 |
-
slot_time,
|
| 2244 |
-
slot_time_scale,
|
| 2245 |
-
) in enumerate(pareto_slots):
|
| 2246 |
-
slot_price_scale.change(
|
| 2247 |
-
_on_pareto_plot_scale(slot_index, "price"),
|
| 2248 |
-
inputs=[
|
| 2249 |
-
dataset_dd,
|
| 2250 |
-
metric_dd,
|
| 2251 |
-
models_dd,
|
| 2252 |
-
slot_price_scale,
|
| 2253 |
-
view_state,
|
| 2254 |
-
],
|
| 2255 |
-
outputs=[slot_price, pareto_all_scale, view_state],
|
| 2256 |
-
**_VIEW_EVENTS,
|
| 2257 |
-
)
|
| 2258 |
-
slot_time_scale.change(
|
| 2259 |
-
_on_pareto_plot_scale(slot_index, "time"),
|
| 2260 |
-
inputs=[
|
| 2261 |
-
dataset_dd,
|
| 2262 |
-
metric_dd,
|
| 2263 |
-
models_dd,
|
| 2264 |
-
slot_time_scale,
|
| 2265 |
-
view_state,
|
| 2266 |
-
],
|
| 2267 |
-
outputs=[slot_time, pareto_all_scale, view_state],
|
| 2268 |
-
**_VIEW_EVENTS,
|
| 2269 |
-
)
|
| 2270 |
-
|
| 2271 |
prompt_count.change(
|
| 2272 |
on_samples_controls,
|
| 2273 |
inputs=[dataset_dd, models_dd, prompt_count, seed_state],
|
|
|
|
| 1 |
from html import escape
|
|
|
|
| 2 |
from pathlib import Path
|
| 3 |
import base64
|
| 4 |
import random
|
|
|
|
| 25 |
_PARETO_SLOT_COUNT = 1 + MAX_PARETO_METRICS * 8
|
| 26 |
_PARETO_PRICE_COLUMN = "Price / Image (USD)"
|
| 27 |
_PARETO_TIME_COLUMN = "Min Generation Time (s)"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
TAB_LEADERBOARDS = "leaderboards"
|
| 30 |
TAB_PARETO = "pareto"
|
|
|
|
| 244 |
return bool(dataset and dataset.get("samples"))
|
| 245 |
|
| 246 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
def _dataset_has_pareto(datasets, dataset_id):
|
| 248 |
dataset = _item(datasets, dataset_id)
|
| 249 |
columns = getattr(dataset.get("data") if dataset else None, "columns", [])
|
|
|
|
| 314 |
]
|
| 315 |
|
| 316 |
|
| 317 |
+
def _model_choices(datasets, dataset_id):
|
| 318 |
cached = _MODEL_CHOICES_CACHE.get(dataset_id)
|
| 319 |
+
if cached is not None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 320 |
return cached
|
| 321 |
+
dataset = _item(datasets, dataset_id)
|
| 322 |
+
data = dataset.get("data") if dataset else None
|
| 323 |
+
if data is None or "Model" not in getattr(data, "columns", []):
|
| 324 |
+
_MODEL_CHOICES_CACHE[dataset_id] = []
|
| 325 |
+
return []
|
| 326 |
+
models = data["Model"].dropna().astype(str).unique().tolist()
|
| 327 |
+
# (label, value) so the UI shows the shared name but filters on the raw id.
|
| 328 |
+
choices = sorted(
|
| 329 |
+
((display_model_name(model), model) for model in models),
|
| 330 |
+
key=lambda item: item[0].casefold(),
|
| 331 |
+
)
|
| 332 |
+
_MODEL_CHOICES_CACHE[dataset_id] = choices
|
| 333 |
+
return choices
|
| 334 |
|
| 335 |
|
| 336 |
def _model_choice_values(choices):
|
|
|
|
| 585 |
"Arena Art Elo": "Art",
|
| 586 |
"Arena Portraits Elo": "Portraits",
|
| 587 |
"Arena Text Rendering Elo": "Text Rendering",
|
| 588 |
+
"Raw Win Rate": "Raw win rate",
|
| 589 |
"Median Generation Time (s)": "Median generation time",
|
| 590 |
"Min Generation Time (s)": "Min generation time",
|
| 591 |
"Price / Image (USD)": "Price per image",
|
|
|
|
| 670 |
x_title,
|
| 671 |
x_hover_prefix="",
|
| 672 |
x_hover_suffix="",
|
|
|
|
| 673 |
):
|
| 674 |
scatter = (
|
| 675 |
data[["Model", score_column, x_column]]
|
|
|
|
| 755 |
)
|
| 756 |
axis_font = {"color": "#fafafa", "size": 13}
|
| 757 |
tick_font = {"color": "#a3a3a3", "size": 12}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 758 |
fig.update_xaxes(
|
|
|
|
|
|
|
| 759 |
showgrid=True,
|
| 760 |
gridcolor="rgba(74, 57, 98, 0.55)",
|
| 761 |
zeroline=False,
|
|
|
|
| 774 |
return fig
|
| 775 |
|
| 776 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 777 |
def _pareto_axis(data, score_column, x_column, x_title, missing_message, empty_message, **hover):
|
| 778 |
if x_column not in data.columns:
|
| 779 |
return None, missing_message
|
|
|
|
| 789 |
return fig, None
|
| 790 |
|
| 791 |
|
| 792 |
+
def _pareto_pair(data, score_column):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 793 |
score_missing = "No score data is available for this metric."
|
| 794 |
if data is None or not score_column or score_column not in data.columns:
|
| 795 |
return None, score_missing, None, score_missing
|
|
|
|
| 802 |
"Price per image isn't available for this dataset.",
|
| 803 |
"No models have both a score and a price for this metric.",
|
| 804 |
x_hover_prefix="$",
|
|
|
|
| 805 |
)
|
| 806 |
time_fig, time_message = _pareto_axis(
|
| 807 |
data,
|
|
|
|
| 811 |
"Min generation time isn't available for this dataset.",
|
| 812 |
"No models have both a score and a min generation time for this metric.",
|
| 813 |
x_hover_suffix="s",
|
|
|
|
| 814 |
)
|
| 815 |
return price_fig, price_message, time_fig, time_message
|
| 816 |
|
|
|
|
| 849 |
return " ".join(notes)
|
| 850 |
|
| 851 |
|
| 852 |
+
def _pareto_slot_updates(data, score_columns):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 853 |
"""Updates for a fixed bank of Gradio Plot slots (visible/hidden)."""
|
| 854 |
score_columns = [column for column in (score_columns or []) if column]
|
|
|
|
|
|
|
| 855 |
has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
|
| 856 |
has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
|
| 857 |
dataset_note = _pareto_dataset_message(data)
|
|
|
|
| 873 |
continue
|
| 874 |
score_column = score_columns[index]
|
| 875 |
price_fig, price_message, time_fig, time_message = _pareto_pair(
|
| 876 |
+
data, score_column
|
|
|
|
|
|
|
|
|
|
| 877 |
)
|
| 878 |
show_price = price_fig is not None
|
| 879 |
show_time = time_fig is not None
|
|
|
|
| 900 |
return updates
|
| 901 |
|
| 902 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 903 |
def _samples_html(samples, selected_models, num_prompts, seed=0):
|
| 904 |
if not samples:
|
| 905 |
return _pareto_unavailable_html(
|
|
|
|
| 1044 |
gr.Markdown(
|
| 1045 |
"<p class='filter-help'>"
|
| 1046 |
"These filters apply to Leaderboards, Pareto plots, and Samples. "
|
| 1047 |
+
"On Samples, only datasets we have generations for are listed. "
|
| 1048 |
+
"On Pareto plots, only datasets with price or generation time "
|
| 1049 |
+
"are listed. Search in Models, or leave it empty to include "
|
| 1050 |
+
"every model."
|
| 1051 |
"</p>",
|
| 1052 |
elem_classes="filter-help-host",
|
| 1053 |
)
|
|
|
|
| 1117 |
) as pp_tab:
|
| 1118 |
gr.Markdown(
|
| 1119 |
"<p class='view-help'>"
|
| 1120 |
+
"Score against price and generation time. Green points are on the "
|
| 1121 |
+
"frontier; lavender points sit below it. Hover a point to see "
|
| 1122 |
+
"which model it is."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1123 |
"</p>",
|
| 1124 |
elem_classes="view-help-host",
|
| 1125 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1126 |
pareto_dataset_note = gr.HTML(
|
| 1127 |
"",
|
| 1128 |
padding=False,
|
|
|
|
| 1148 |
min_width=320,
|
| 1149 |
elem_classes="pareto-col",
|
| 1150 |
) as slot_price_col:
|
| 1151 |
+
gr.Markdown(
|
| 1152 |
+
"#### Price vs score",
|
| 1153 |
+
elem_classes="pareto-subhead",
|
| 1154 |
)
|
| 1155 |
slot_price = gr.Plot(
|
| 1156 |
value=None,
|
|
|
|
| 1162 |
min_width=320,
|
| 1163 |
elem_classes="pareto-col",
|
| 1164 |
) as slot_time_col:
|
| 1165 |
+
gr.Markdown(
|
| 1166 |
+
"#### Min generation time vs score",
|
| 1167 |
+
elem_classes="pareto-subhead",
|
| 1168 |
)
|
| 1169 |
slot_time = gr.Plot(
|
| 1170 |
value=None,
|
|
|
|
| 1185 |
slot_layout,
|
| 1186 |
slot_price_col,
|
| 1187 |
slot_price,
|
|
|
|
| 1188 |
slot_time_col,
|
| 1189 |
slot_time,
|
|
|
|
| 1190 |
)
|
| 1191 |
)
|
| 1192 |
|
|
|
|
| 1235 |
with gr.TabItem("About", id=TAB_ABOUT) as about_tab:
|
| 1236 |
render_about()
|
| 1237 |
|
| 1238 |
+
def _synced_filters(dataset_id, metric_id, models, *, clear_metric=False):
|
|
|
|
|
|
|
| 1239 |
if clear_metric:
|
| 1240 |
metric_id = []
|
| 1241 |
else:
|
| 1242 |
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
| 1243 |
+
model_choices = _model_choices(datasets, dataset_id)
|
|
|
|
|
|
|
| 1244 |
model_values = set(_model_choice_values(model_choices))
|
| 1245 |
models = [model for model in (models or []) if model in model_values]
|
| 1246 |
metric_choices = _metric_dropdown_choices(datasets, metrics, dataset_id)
|
|
|
|
| 1324 |
"optimized": list(
|
| 1325 |
extras.get("optimized", prev.get("optimized") or [])
|
| 1326 |
),
|
|
|
|
|
|
|
| 1327 |
"stale": {
|
| 1328 |
TAB_LEADERBOARDS: not flags["include_leaderboard"],
|
| 1329 |
TAB_PARETO: not flags["include_pareto"],
|
|
|
|
| 1383 |
include_leaderboard=True,
|
| 1384 |
include_pareto=False,
|
| 1385 |
include_samples=False,
|
|
|
|
|
|
|
| 1386 |
):
|
| 1387 |
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 1388 |
data = view["data"]
|
|
|
|
| 1403 |
ranking_html = gr.skip()
|
| 1404 |
if include_pareto:
|
| 1405 |
pareto_data = _filter_leaderboard(data, [], [], [], models=models)
|
| 1406 |
+
pareto_updates = _pareto_slot_updates(pareto_data, view["score_columns"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1407 |
else:
|
| 1408 |
pareto_updates = _pareto_skip_updates()
|
| 1409 |
if include_samples:
|
|
|
|
| 1445 |
dataset_changed = source == "dataset" and dataset_id != view_state.get(
|
| 1446 |
"dataset_id"
|
| 1447 |
)
|
| 1448 |
+
|
|
|
|
|
|
|
| 1449 |
if source == "dataset":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1450 |
synced = _synced_filters(
|
| 1451 |
+
dataset_id, metric_id, models, clear_metric=dataset_changed
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1452 |
)
|
| 1453 |
dataset_id, metric_id, models = synced[:3]
|
| 1454 |
metric_update, models_update = synced[3], synced[4]
|
|
|
|
| 1486 |
):
|
| 1487 |
return None
|
| 1488 |
|
| 1489 |
+
selected_tab = tab
|
| 1490 |
extras = (
|
| 1491 |
list(platform_value or []),
|
| 1492 |
list(owner_value or []),
|
| 1493 |
list(optimized_value or []),
|
| 1494 |
)
|
| 1495 |
extra_updates = None
|
| 1496 |
+
can_pareto = _dataset_has_pareto(datasets, dataset_id)
|
| 1497 |
+
can_samples = _dataset_has_samples(datasets, dataset_id)
|
| 1498 |
if source == "dataset":
|
| 1499 |
+
if tab == TAB_SAMPLES and not can_samples:
|
| 1500 |
+
selected_tab = TAB_LEADERBOARDS
|
| 1501 |
+
elif tab == TAB_PARETO and not can_pareto:
|
| 1502 |
+
selected_tab = TAB_LEADERBOARDS
|
| 1503 |
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 1504 |
extra_updates = _leaderboard_extras(
|
| 1505 |
view["data"] if view else None,
|
|
|
|
| 1537 |
extras[2],
|
| 1538 |
num_prompts,
|
| 1539 |
seed,
|
|
|
|
|
|
|
| 1540 |
**flags,
|
| 1541 |
),
|
| 1542 |
"state": _commit_state(
|
|
|
|
| 1677 |
)
|
| 1678 |
dataset_update = _dataset_dropdown_update(datasets, tab, dataset_id)
|
| 1679 |
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
| 1680 |
+
models = list(models or [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1681 |
view_state["current_tab"] = tab
|
| 1682 |
view_state["dataset_id"] = dataset_id
|
| 1683 |
view_state["metric_id"] = metric_id
|
|
|
|
| 1714 |
filters_vis,
|
| 1715 |
dataset_update,
|
| 1716 |
metric_vis,
|
| 1717 |
+
gr.skip(),
|
| 1718 |
*lb_filters,
|
| 1719 |
)
|
| 1720 |
tab_select = (
|
|
|
|
| 1739 |
optimized_value,
|
| 1740 |
num_prompts,
|
| 1741 |
seed,
|
|
|
|
|
|
|
| 1742 |
**flags,
|
| 1743 |
)
|
| 1744 |
stale[tab] = False
|
|
|
|
| 1795 |
next_seed,
|
| 1796 |
)
|
| 1797 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1798 |
def _on_tab(tab):
|
| 1799 |
def handler(
|
| 1800 |
dataset_id,
|
|
|
|
| 1832 |
"platform": [],
|
| 1833 |
"owner": [],
|
| 1834 |
"optimized": [],
|
|
|
|
|
|
|
| 1835 |
"stale": {
|
| 1836 |
TAB_LEADERBOARDS: False,
|
| 1837 |
TAB_PARETO: True,
|
|
|
|
| 1843 |
pareto_dataset_note,
|
| 1844 |
*[
|
| 1845 |
component
|
| 1846 |
+
for slot_group, slot_title, slot_note, slot_layout, slot_price_col, slot_price, slot_time_col, slot_time in pareto_slots
|
| 1847 |
for component in (
|
| 1848 |
slot_group,
|
| 1849 |
slot_title,
|
|
|
|
| 1856 |
)
|
| 1857 |
],
|
| 1858 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1859 |
view_inputs = [
|
| 1860 |
platform,
|
| 1861 |
owner,
|
|
|
|
| 1960 |
show_progress="hidden",
|
| 1961 |
)
|
| 1962 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1963 |
prompt_count.change(
|
| 1964 |
on_samples_controls,
|
| 1965 |
inputs=[dataset_dd, models_dd, prompt_count, seed_state],
|