Update page
Browse files- app.js +36 -8
- config.json +2 -2
- data/cosmos-reason2.csv +0 -6
- style.css +6 -2
app.js
CHANGED
|
@@ -478,7 +478,8 @@ function buildTables(filtered, chartsShown) {
|
|
| 478 |
// Resolve table_sort: family-specific overrides global
|
| 479 |
const familyCfg = config.model_families?.[filters.family] || {};
|
| 480 |
const sortRules = familyCfg.table_sort || config.table_sort || [];
|
| 481 |
-
const tableGroupBy = familyCfg.table_group_by || "";
|
|
|
|
| 482 |
|
| 483 |
groupVals.forEach(gv => {
|
| 484 |
const rows = filtered.filter(r => String(r[GROUP_BY]) === String(gv));
|
|
@@ -521,33 +522,60 @@ function buildTables(filtered, chartsShown) {
|
|
| 521 |
|
| 522 |
let html = chartsShown ? '' : `<h3>${heading}</h3>`;
|
| 523 |
html += `<div class="table-scroll"><table><thead><tr>`;
|
| 524 |
-
|
|
|
|
| 525 |
const tip = c.description ? ` data-tip="${c.description.replace(/"/g, '"')}"` : '';
|
| 526 |
-
|
|
|
|
| 527 |
}).join("");
|
| 528 |
html += `</tr></thead><tbody>`;
|
| 529 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 530 |
rows.forEach(r => {
|
| 531 |
const oom = isOOMRow(r);
|
| 532 |
let rowClass = "";
|
| 533 |
-
if (
|
| 534 |
-
const curVal =
|
| 535 |
if (prevGroupVal !== undefined && curVal !== prevGroupVal) {
|
| 536 |
rowClass = "row-group-break";
|
| 537 |
}
|
| 538 |
prevGroupVal = curVal;
|
| 539 |
}
|
| 540 |
html += `<tr class="${rowClass}">`;
|
| 541 |
-
colDefs.forEach(c => {
|
| 542 |
const val = r[c.key];
|
|
|
|
| 543 |
if (c.isModel) {
|
| 544 |
const hfUrl = LINK_PREFIX + val;
|
| 545 |
const modelColor = MODEL_COLORS[val]?.border || '#888';
|
| 546 |
html += `<td class="model-cell"><span class="model-dot" style="background:${modelColor}"></span><a href="${hfUrl}" target="_blank" rel="noopener" style="color:${modelColor}">${val}</a></td>`;
|
| 547 |
} else if (oom) {
|
| 548 |
-
html += `<td><span class="oom">OOM</span></td>`;
|
| 549 |
} else if (c.isMetric) {
|
| 550 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 551 |
} else {
|
| 552 |
html += `<td>${val || "—"}</td>`;
|
| 553 |
}
|
|
|
|
| 478 |
// Resolve table_sort: family-specific overrides global
|
| 479 |
const familyCfg = config.model_families?.[filters.family] || {};
|
| 480 |
const sortRules = familyCfg.table_sort || config.table_sort || [];
|
| 481 |
+
const tableGroupBy = familyCfg.table_group_by || config.table_group_by || "";
|
| 482 |
+
const tableGroupCols = Array.isArray(tableGroupBy) ? tableGroupBy : (tableGroupBy ? [tableGroupBy] : []);
|
| 483 |
|
| 484 |
groupVals.forEach(gv => {
|
| 485 |
const rows = filtered.filter(r => String(r[GROUP_BY]) === String(gv));
|
|
|
|
| 522 |
|
| 523 |
let html = chartsShown ? '' : `<h3>${heading}</h3>`;
|
| 524 |
html += `<div class="table-scroll"><table><thead><tr>`;
|
| 525 |
+
const firstMetricIdx = colDefs.findIndex(c => c.isMetric);
|
| 526 |
+
html += colDefs.map((c, i) => {
|
| 527 |
const tip = c.description ? ` data-tip="${c.description.replace(/"/g, '"')}"` : '';
|
| 528 |
+
const cls = i === firstMetricIdx ? ' class="first-metric"' : '';
|
| 529 |
+
return `<th${tip}${cls}>${c.label}</th>`;
|
| 530 |
}).join("");
|
| 531 |
html += `</tr></thead><tbody>`;
|
| 532 |
|
| 533 |
+
// Compute best metric value per sub-group (tableGroupBy) per column
|
| 534 |
+
const bestByGroup = {};
|
| 535 |
+
const groupRowKey = r => tableGroupCols.length
|
| 536 |
+
? tableGroupCols.map(c => String(r[c] ?? "")).join("\0")
|
| 537 |
+
: "__all__";
|
| 538 |
+
const subGroups = tableGroupCols.length
|
| 539 |
+
? [...new Set(rows.map(groupRowKey))]
|
| 540 |
+
: ["__all__"];
|
| 541 |
+
subGroups.forEach(sg => {
|
| 542 |
+
const groupRows = tableGroupCols.length ? rows.filter(r => groupRowKey(r) === sg) : rows;
|
| 543 |
+
bestByGroup[sg] = {};
|
| 544 |
+
colDefs.filter(c => c.isMetric).forEach(c => {
|
| 545 |
+
const metricCfg = config.metrics.find(m => m.column === c.key);
|
| 546 |
+
const vals = groupRows.map(r => r[c.key]).filter(v => v !== null && v !== undefined);
|
| 547 |
+
if (vals.length) {
|
| 548 |
+
bestByGroup[sg][c.key] = metricCfg?.higher_is_better ? Math.max(...vals) : Math.min(...vals);
|
| 549 |
+
}
|
| 550 |
+
});
|
| 551 |
+
});
|
| 552 |
+
|
| 553 |
rows.forEach(r => {
|
| 554 |
const oom = isOOMRow(r);
|
| 555 |
let rowClass = "";
|
| 556 |
+
if (tableGroupCols.length) {
|
| 557 |
+
const curVal = groupRowKey(r);
|
| 558 |
if (prevGroupVal !== undefined && curVal !== prevGroupVal) {
|
| 559 |
rowClass = "row-group-break";
|
| 560 |
}
|
| 561 |
prevGroupVal = curVal;
|
| 562 |
}
|
| 563 |
html += `<tr class="${rowClass}">`;
|
| 564 |
+
colDefs.forEach((c, i) => {
|
| 565 |
const val = r[c.key];
|
| 566 |
+
const fmCls = i === firstMetricIdx ? ' class="first-metric"' : '';
|
| 567 |
if (c.isModel) {
|
| 568 |
const hfUrl = LINK_PREFIX + val;
|
| 569 |
const modelColor = MODEL_COLORS[val]?.border || '#888';
|
| 570 |
html += `<td class="model-cell"><span class="model-dot" style="background:${modelColor}"></span><a href="${hfUrl}" target="_blank" rel="noopener" style="color:${modelColor}">${val}</a></td>`;
|
| 571 |
} else if (oom) {
|
| 572 |
+
html += `<td${fmCls}><span class="oom">OOM</span></td>`;
|
| 573 |
} else if (c.isMetric) {
|
| 574 |
+
const sg = groupRowKey(r);
|
| 575 |
+
const isBest = val !== null && val !== undefined && val === bestByGroup[sg]?.[c.key];
|
| 576 |
+
const display = val === null ? '<span class="oom">OOM</span>' : (typeof val === "number" ? val.toFixed(2) : (val ?? "—"));
|
| 577 |
+
const modelColor = MODEL_COLORS[r[MODEL_COL]]?.border || '#888';
|
| 578 |
+
html += `<td${fmCls}>${isBest ? '<strong style="color:' + "white" + '; opacity: 0.7">' + display + '</strong>' : display}</td>`;
|
| 579 |
} else {
|
| 580 |
html += `<td>${val || "—"}</td>`;
|
| 581 |
}
|
config.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"title": "Edge Inference Benchmarks",
|
| 3 |
-
"subtitle": "Compare throughput and latency across devices and model variants.
|
| 4 |
"model_column": "model",
|
| 5 |
"model_family_column": "model_family",
|
| 6 |
"model_link_prefix": "https://huggingface.co/",
|
|
@@ -129,7 +129,7 @@
|
|
| 129 |
"model_families": {
|
| 130 |
"Cosmos-Reason2-2B": {
|
| 131 |
"data_file": "data/cosmos-reason2.csv",
|
| 132 |
-
"table_group_by": "res",
|
| 133 |
"experiment_setup": {
|
| 134 |
"agx_thor": "Measurement setup: NVIDIA vLLM 26.01, 256 tokens generated, 10 warm-up runs, averaged over 25 runs.",
|
| 135 |
"agx_orin": "Measurement setup: NVIDIA vLLM 0.14.0 for Jetson, 256 tokens generated, 10 warm-up runs, averaged over 25 runs.",
|
|
|
|
| 1 |
{
|
| 2 |
"title": "Edge Inference Benchmarks",
|
| 3 |
+
"subtitle": "Compare throughput and latency across devices and model variants.",
|
| 4 |
"model_column": "model",
|
| 5 |
"model_family_column": "model_family",
|
| 6 |
"model_link_prefix": "https://huggingface.co/",
|
|
|
|
| 129 |
"model_families": {
|
| 130 |
"Cosmos-Reason2-2B": {
|
| 131 |
"data_file": "data/cosmos-reason2.csv",
|
| 132 |
+
"table_group_by": ["res", "fps"],
|
| 133 |
"experiment_setup": {
|
| 134 |
"agx_thor": "Measurement setup: NVIDIA vLLM 26.01, 256 tokens generated, 10 warm-up runs, averaged over 25 runs.",
|
| 135 |
"agx_orin": "Measurement setup: NVIDIA vLLM 0.14.0 for Jetson, 256 tokens generated, 10 warm-up runs, averaged over 25 runs.",
|
data/cosmos-reason2.csv
CHANGED
|
@@ -62,9 +62,6 @@ Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,agx_thor,854x480,2,6,3.
|
|
| 62 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,agx_thor,854x480,4,12,3.9974,512.34,11.43,349.00
|
| 63 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1280x720,2,6,7.7657,263.72,19.47,918.23
|
| 64 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1280x720,4,12,10.2787,199.25,29.04,918.25
|
| 65 |
-
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1920x1080,2,6,OOM,OOM,OOM,OOM
|
| 66 |
-
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1920x1080,4,12,OOM,OOM,OOM,OOM
|
| 67 |
-
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,854x480,4,12,OOM,OOM,OOM,OOM
|
| 68 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,1280x720,N/A,N/A,3.5585,71.94,11.69,57.02
|
| 69 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,1920x1080,N/A,N/A,3.4224,74.80,11.68,54.49
|
| 70 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,854x480,N/A,N/A,3.5028,73.08,11.67,56.27
|
|
@@ -154,9 +151,6 @@ Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,1,agx_thor,854x480,2,6,5.0284,5
|
|
| 154 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,1,agx_thor,854x480,4,12,4.5504,56.26,17.01,123.61
|
| 155 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,1,orin_nano_super,OOM,OOM,OOM,OOM,OOM,OOM,OOM
|
| 156 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_orin,1280x720,2,6,10.2747,199.33,29.03,918.06
|
| 157 |
-
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_orin,1280x720,4,12,OOM,OOM,OOM,OOM
|
| 158 |
-
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_orin,1920x1080,2,6,OOM,OOM,OOM,OOM
|
| 159 |
-
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_orin,1920x1080,4,12,OOM,OOM,OOM,OOM
|
| 160 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1280x720,2,6,7.0704,289.66,19.80,778.89
|
| 161 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1280x720,4,12,7.0242,291.56,19.61,779.39
|
| 162 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1920x1080,2,6,10.1481,201.81,22.92,1749.06
|
|
|
|
| 62 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,agx_thor,854x480,4,12,3.9974,512.34,11.43,349.00
|
| 63 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1280x720,2,6,7.7657,263.72,19.47,918.23
|
| 64 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1280x720,4,12,10.2787,199.25,29.04,918.25
|
|
|
|
|
|
|
|
|
|
| 65 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,1280x720,N/A,N/A,3.5585,71.94,11.69,57.02
|
| 66 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,1920x1080,N/A,N/A,3.4224,74.80,11.68,54.49
|
| 67 |
Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,854x480,N/A,N/A,3.5028,73.08,11.67,56.27
|
|
|
|
| 151 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,1,agx_thor,854x480,4,12,4.5504,56.26,17.01,123.61
|
| 152 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,1,orin_nano_super,OOM,OOM,OOM,OOM,OOM,OOM,OOM
|
| 153 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_orin,1280x720,2,6,10.2747,199.33,29.03,918.06
|
|
|
|
|
|
|
|
|
|
| 154 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1280x720,2,6,7.0704,289.66,19.80,778.89
|
| 155 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1280x720,4,12,7.0242,291.56,19.61,779.39
|
| 156 |
Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1920x1080,2,6,10.1481,201.81,22.92,1749.06
|
style.css
CHANGED
|
@@ -13,7 +13,7 @@
|
|
| 13 |
--text: #e8e8e8;
|
| 14 |
--text-muted: #8899aa;
|
| 15 |
--text-dim: #5a6a7a;
|
| 16 |
-
--border: rgba(255, 255, 255, 0.
|
| 17 |
--btn-hover-bg: rgba(255, 255, 255, 0.03);
|
| 18 |
--btn-active-bg: rgba(88, 177, 195, 0.12);
|
| 19 |
--btn-active-border: rgba(88, 177, 195, 0.3);
|
|
@@ -316,7 +316,7 @@ tbody td {
|
|
| 316 |
}
|
| 317 |
|
| 318 |
tbody tr:last-child td {
|
| 319 |
-
border-bottom:
|
| 320 |
}
|
| 321 |
|
| 322 |
tbody tr:hover {
|
|
@@ -348,6 +348,10 @@ tbody tr:hover {
|
|
| 348 |
flex-shrink: 0;
|
| 349 |
}
|
| 350 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 351 |
.oom {
|
| 352 |
color: var(--red);
|
| 353 |
font-weight: 600;
|
|
|
|
| 13 |
--text: #e8e8e8;
|
| 14 |
--text-muted: #8899aa;
|
| 15 |
--text-dim: #5a6a7a;
|
| 16 |
+
--border: rgba(255, 255, 255, 0.15);
|
| 17 |
--btn-hover-bg: rgba(255, 255, 255, 0.03);
|
| 18 |
--btn-active-bg: rgba(88, 177, 195, 0.12);
|
| 19 |
--btn-active-border: rgba(88, 177, 195, 0.3);
|
|
|
|
| 316 |
}
|
| 317 |
|
| 318 |
tbody tr:last-child td {
|
| 319 |
+
border-bottom: 1px solid var(--border);
|
| 320 |
}
|
| 321 |
|
| 322 |
tbody tr:hover {
|
|
|
|
| 348 |
flex-shrink: 0;
|
| 349 |
}
|
| 350 |
|
| 351 |
+
th.first-metric, td.first-metric {
|
| 352 |
+
padding-left: 2.5rem;
|
| 353 |
+
}
|
| 354 |
+
|
| 355 |
.oom {
|
| 356 |
color: var(--red);
|
| 357 |
font-weight: 600;
|