JonnaMat commited on
Commit
32853d9
·
verified ·
1 Parent(s): 979f9e7

Update page

Browse files
Files changed (4) hide show
  1. app.js +36 -8
  2. config.json +2 -2
  3. data/cosmos-reason2.csv +0 -6
  4. style.css +6 -2
app.js CHANGED
@@ -478,7 +478,8 @@ function buildTables(filtered, chartsShown) {
478
  // Resolve table_sort: family-specific overrides global
479
  const familyCfg = config.model_families?.[filters.family] || {};
480
  const sortRules = familyCfg.table_sort || config.table_sort || [];
481
- const tableGroupBy = familyCfg.table_group_by || "";
 
482
 
483
  groupVals.forEach(gv => {
484
  const rows = filtered.filter(r => String(r[GROUP_BY]) === String(gv));
@@ -521,33 +522,60 @@ function buildTables(filtered, chartsShown) {
521
 
522
  let html = chartsShown ? '' : `<h3>${heading}</h3>`;
523
  html += `<div class="table-scroll"><table><thead><tr>`;
524
- html += colDefs.map(c => {
 
525
  const tip = c.description ? ` data-tip="${c.description.replace(/"/g, '&quot;')}"` : '';
526
- return `<th${tip}>${c.label}</th>`;
 
527
  }).join("");
528
  html += `</tr></thead><tbody>`;
529
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
530
  rows.forEach(r => {
531
  const oom = isOOMRow(r);
532
  let rowClass = "";
533
- if (tableGroupBy) {
534
- const curVal = String(r[tableGroupBy] ?? "");
535
  if (prevGroupVal !== undefined && curVal !== prevGroupVal) {
536
  rowClass = "row-group-break";
537
  }
538
  prevGroupVal = curVal;
539
  }
540
  html += `<tr class="${rowClass}">`;
541
- colDefs.forEach(c => {
542
  const val = r[c.key];
 
543
  if (c.isModel) {
544
  const hfUrl = LINK_PREFIX + val;
545
  const modelColor = MODEL_COLORS[val]?.border || '#888';
546
  html += `<td class="model-cell"><span class="model-dot" style="background:${modelColor}"></span><a href="${hfUrl}" target="_blank" rel="noopener" style="color:${modelColor}">${val}</a></td>`;
547
  } else if (oom) {
548
- html += `<td><span class="oom">OOM</span></td>`;
549
  } else if (c.isMetric) {
550
- html += `<td>${val === null ? '<span class="oom">OOM</span>' : (val ?? "—")}</td>`;
 
 
 
 
551
  } else {
552
  html += `<td>${val || "—"}</td>`;
553
  }
 
478
  // Resolve table_sort: family-specific overrides global
479
  const familyCfg = config.model_families?.[filters.family] || {};
480
  const sortRules = familyCfg.table_sort || config.table_sort || [];
481
+ const tableGroupBy = familyCfg.table_group_by || config.table_group_by || "";
482
+ const tableGroupCols = Array.isArray(tableGroupBy) ? tableGroupBy : (tableGroupBy ? [tableGroupBy] : []);
483
 
484
  groupVals.forEach(gv => {
485
  const rows = filtered.filter(r => String(r[GROUP_BY]) === String(gv));
 
522
 
523
  let html = chartsShown ? '' : `<h3>${heading}</h3>`;
524
  html += `<div class="table-scroll"><table><thead><tr>`;
525
+ const firstMetricIdx = colDefs.findIndex(c => c.isMetric);
526
+ html += colDefs.map((c, i) => {
527
  const tip = c.description ? ` data-tip="${c.description.replace(/"/g, '&quot;')}"` : '';
528
+ const cls = i === firstMetricIdx ? ' class="first-metric"' : '';
529
+ return `<th${tip}${cls}>${c.label}</th>`;
530
  }).join("");
531
  html += `</tr></thead><tbody>`;
532
 
533
+ // Compute best metric value per sub-group (tableGroupBy) per column
534
+ const bestByGroup = {};
535
+ const groupRowKey = r => tableGroupCols.length
536
+ ? tableGroupCols.map(c => String(r[c] ?? "")).join("\0")
537
+ : "__all__";
538
+ const subGroups = tableGroupCols.length
539
+ ? [...new Set(rows.map(groupRowKey))]
540
+ : ["__all__"];
541
+ subGroups.forEach(sg => {
542
+ const groupRows = tableGroupCols.length ? rows.filter(r => groupRowKey(r) === sg) : rows;
543
+ bestByGroup[sg] = {};
544
+ colDefs.filter(c => c.isMetric).forEach(c => {
545
+ const metricCfg = config.metrics.find(m => m.column === c.key);
546
+ const vals = groupRows.map(r => r[c.key]).filter(v => v !== null && v !== undefined);
547
+ if (vals.length) {
548
+ bestByGroup[sg][c.key] = metricCfg?.higher_is_better ? Math.max(...vals) : Math.min(...vals);
549
+ }
550
+ });
551
+ });
552
+
553
  rows.forEach(r => {
554
  const oom = isOOMRow(r);
555
  let rowClass = "";
556
+ if (tableGroupCols.length) {
557
+ const curVal = groupRowKey(r);
558
  if (prevGroupVal !== undefined && curVal !== prevGroupVal) {
559
  rowClass = "row-group-break";
560
  }
561
  prevGroupVal = curVal;
562
  }
563
  html += `<tr class="${rowClass}">`;
564
+ colDefs.forEach((c, i) => {
565
  const val = r[c.key];
566
+ const fmCls = i === firstMetricIdx ? ' class="first-metric"' : '';
567
  if (c.isModel) {
568
  const hfUrl = LINK_PREFIX + val;
569
  const modelColor = MODEL_COLORS[val]?.border || '#888';
570
  html += `<td class="model-cell"><span class="model-dot" style="background:${modelColor}"></span><a href="${hfUrl}" target="_blank" rel="noopener" style="color:${modelColor}">${val}</a></td>`;
571
  } else if (oom) {
572
+ html += `<td${fmCls}><span class="oom">OOM</span></td>`;
573
  } else if (c.isMetric) {
574
+ const sg = groupRowKey(r);
575
+ const isBest = val !== null && val !== undefined && val === bestByGroup[sg]?.[c.key];
576
+ const display = val === null ? '<span class="oom">OOM</span>' : (typeof val === "number" ? val.toFixed(2) : (val ?? "—"));
577
+ const modelColor = MODEL_COLORS[r[MODEL_COL]]?.border || '#888';
578
+ html += `<td${fmCls}>${isBest ? '<strong style="color:' + "white" + '; opacity: 0.7">' + display + '</strong>' : display}</td>`;
579
  } else {
580
  html += `<td>${val || "—"}</td>`;
581
  }
config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "title": "Edge Inference Benchmarks",
3
- "subtitle": "Compare throughput and latency across devices and model variants. Use filters to focus on what matters.",
4
  "model_column": "model",
5
  "model_family_column": "model_family",
6
  "model_link_prefix": "https://huggingface.co/",
@@ -129,7 +129,7 @@
129
  "model_families": {
130
  "Cosmos-Reason2-2B": {
131
  "data_file": "data/cosmos-reason2.csv",
132
- "table_group_by": "res",
133
  "experiment_setup": {
134
  "agx_thor": "Measurement setup: NVIDIA vLLM 26.01, 256 tokens generated, 10 warm-up runs, averaged over 25 runs.",
135
  "agx_orin": "Measurement setup: NVIDIA vLLM 0.14.0 for Jetson, 256 tokens generated, 10 warm-up runs, averaged over 25 runs.",
 
1
  {
2
  "title": "Edge Inference Benchmarks",
3
+ "subtitle": "Compare throughput and latency across devices and model variants.",
4
  "model_column": "model",
5
  "model_family_column": "model_family",
6
  "model_link_prefix": "https://huggingface.co/",
 
129
  "model_families": {
130
  "Cosmos-Reason2-2B": {
131
  "data_file": "data/cosmos-reason2.csv",
132
+ "table_group_by": ["res", "fps"],
133
  "experiment_setup": {
134
  "agx_thor": "Measurement setup: NVIDIA vLLM 26.01, 256 tokens generated, 10 warm-up runs, averaged over 25 runs.",
135
  "agx_orin": "Measurement setup: NVIDIA vLLM 0.14.0 for Jetson, 256 tokens generated, 10 warm-up runs, averaged over 25 runs.",
data/cosmos-reason2.csv CHANGED
@@ -62,9 +62,6 @@ Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,agx_thor,854x480,2,6,3.
62
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,agx_thor,854x480,4,12,3.9974,512.34,11.43,349.00
63
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1280x720,2,6,7.7657,263.72,19.47,918.23
64
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1280x720,4,12,10.2787,199.25,29.04,918.25
65
- Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1920x1080,2,6,OOM,OOM,OOM,OOM
66
- Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1920x1080,4,12,OOM,OOM,OOM,OOM
67
- Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,854x480,4,12,OOM,OOM,OOM,OOM
68
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,1280x720,N/A,N/A,3.5585,71.94,11.69,57.02
69
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,1920x1080,N/A,N/A,3.4224,74.80,11.68,54.49
70
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,854x480,N/A,N/A,3.5028,73.08,11.67,56.27
@@ -154,9 +151,6 @@ Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,1,agx_thor,854x480,2,6,5.0284,5
154
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,1,agx_thor,854x480,4,12,4.5504,56.26,17.01,123.61
155
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,1,orin_nano_super,OOM,OOM,OOM,OOM,OOM,OOM,OOM
156
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_orin,1280x720,2,6,10.2747,199.33,29.03,918.06
157
- Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_orin,1280x720,4,12,OOM,OOM,OOM,OOM
158
- Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_orin,1920x1080,2,6,OOM,OOM,OOM,OOM
159
- Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_orin,1920x1080,4,12,OOM,OOM,OOM,OOM
160
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1280x720,2,6,7.0704,289.66,19.80,778.89
161
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1280x720,4,12,7.0242,291.56,19.61,779.39
162
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1920x1080,2,6,10.1481,201.81,22.92,1749.06
 
62
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,agx_thor,854x480,4,12,3.9974,512.34,11.43,349.00
63
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1280x720,2,6,7.7657,263.72,19.47,918.23
64
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16,video,8,orin_nano_super,1280x720,4,12,10.2787,199.25,29.04,918.25
 
 
 
65
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,1280x720,N/A,N/A,3.5585,71.94,11.69,57.02
66
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,1920x1080,N/A,N/A,3.4224,74.80,11.68,54.49
67
  Cosmos-Reason2-2B,embedl/Cosmos-Reason2-2B-W4A16-Edge2,image,1,agx_orin,854x480,N/A,N/A,3.5028,73.08,11.67,56.27
 
151
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,1,agx_thor,854x480,4,12,4.5504,56.26,17.01,123.61
152
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,1,orin_nano_super,OOM,OOM,OOM,OOM,OOM,OOM,OOM
153
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_orin,1280x720,2,6,10.2747,199.33,29.03,918.06
 
 
 
154
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1280x720,2,6,7.0704,289.66,19.80,778.89
155
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1280x720,4,12,7.0242,291.56,19.61,779.39
156
  Cosmos-Reason2-2B,nvidia/Cosmos-Reason2-2B,video,8,agx_thor,1920x1080,2,6,10.1481,201.81,22.92,1749.06
style.css CHANGED
@@ -13,7 +13,7 @@
13
  --text: #e8e8e8;
14
  --text-muted: #8899aa;
15
  --text-dim: #5a6a7a;
16
- --border: rgba(255, 255, 255, 0.06);
17
  --btn-hover-bg: rgba(255, 255, 255, 0.03);
18
  --btn-active-bg: rgba(88, 177, 195, 0.12);
19
  --btn-active-border: rgba(88, 177, 195, 0.3);
@@ -316,7 +316,7 @@ tbody td {
316
  }
317
 
318
  tbody tr:last-child td {
319
- border-bottom: none;
320
  }
321
 
322
  tbody tr:hover {
@@ -348,6 +348,10 @@ tbody tr:hover {
348
  flex-shrink: 0;
349
  }
350
 
 
 
 
 
351
  .oom {
352
  color: var(--red);
353
  font-weight: 600;
 
13
  --text: #e8e8e8;
14
  --text-muted: #8899aa;
15
  --text-dim: #5a6a7a;
16
+ --border: rgba(255, 255, 255, 0.15);
17
  --btn-hover-bg: rgba(255, 255, 255, 0.03);
18
  --btn-active-bg: rgba(88, 177, 195, 0.12);
19
  --btn-active-border: rgba(88, 177, 195, 0.3);
 
316
  }
317
 
318
  tbody tr:last-child td {
319
+ border-bottom: 1px solid var(--border);
320
  }
321
 
322
  tbody tr:hover {
 
348
  flex-shrink: 0;
349
  }
350
 
351
+ th.first-metric, td.first-metric {
352
+ padding-left: 2.5rem;
353
+ }
354
+
355
  .oom {
356
  color: var(--red);
357
  font-weight: 600;