Add deploy-benchmark model families (8 models, lat/ips/mem metrics)

app.js

@@ -30,7 +30,9 @@ function parseCSV(text) {
         headers.forEach((h, i) => {
             const raw = (vals[i] || "").trim();
             if (raw === "") {
-                row[h] = numericCols.has(h) ? null : "";
+                // Empty numeric cell = not measured (renders as a dash);
+                // only an explicit OOM value maps to null.
+                row[h] = numericCols.has(h) ? undefined : "";
             } else if (numericCols.has(h)) {
                 const upper = raw.toUpperCase();
                 row[h] = upper === "OOM" ? null : upper === "N/A" ? "N/A" : parseFloat(raw);
@@ -245,7 +247,11 @@ function parseModelSize(s) {
 }
 
 function isOOMRow(row) {
-    return config.metrics.every(m => row[m.column] === null);
+    // Only consider metric columns present in this row's CSV — families use
+    // disjoint metric sets (LLM tps/... vs deploy lat/ips/mem), and absent
+    // columns (undefined) must not affect OOM detection.
+    const present = config.metrics.filter(m => row[m.column] !== undefined);
+    return present.length > 0 && present.every(m => row[m.column] === null);
 }
 
 function getActiveRows() {
@@ -288,7 +294,8 @@ function modelCellHtml(model) {
 
 function metricCellHtml(val, isBest, extraClass) {
     const cls = extraClass ? ` class="${extraClass}"` : ' class="metric-cell"';
-    if (val === null || val === undefined) return `<td${cls}><span class="oom">OOM</span></td>`;
+    if (val === undefined || val === "") return `<td${cls}>\u2014</td>`;
+    if (val === null) return `<td${cls}><span class="oom">OOM</span></td>`;
     const display = typeof val === "number" ? val.toFixed(2) : (val || "—");
     const content = isBest ? `<strong style="color: white; opacity: 0.7">${display}</strong>` : display;
     return `<td${cls}>${content}</td>`;
@@ -514,7 +521,7 @@ function buildChart(filtered) {
 
     // Only show metric buttons for metrics that have non-zero data
     const chartVisibleMetrics = config.metrics.filter(m =>
-        gRows.some(r => r[m.column] !== null && r[m.column] !== 0 && r[m.column] !== "N/A")
+        gRows.some(r => r[m.column] !== undefined && r[m.column] !== null && r[m.column] !== 0 && r[m.column] !== "N/A")
     );
     if (chartVisibleMetrics.length > 1) {
         const metricEl = metricGroup.querySelector(".btn-group");
@@ -668,7 +675,7 @@ function buildTables(filtered, chartsShown) {
 
     // Hide metric columns where every value in the filtered data is zero, null, or N/A
     const visibleMetrics = config.metrics.filter(m =>
-        filtered.some(r => r[m.column] !== null && r[m.column] !== 0 && r[m.column] !== "N/A")
+        filtered.some(r => r[m.column] !== undefined && r[m.column] !== null && r[m.column] !== 0 && r[m.column] !== "N/A")
     );
 
     // Build column list: Model + visible display cols + metrics
@@ -872,11 +879,13 @@ async function buildAccuracyTable() {
     const card = document.createElement("div");
     card.className = "table-card";
 
-    // Find best value per column (higher is better for accuracy)
+    // Find best value per column. Direction defaults to higher-is-better;
+    // families can set accuracy_higher_is_better: false (e.g. WER).
+    const higherIsBetter = familyCfg.accuracy_higher_is_better !== false;
     const best = {};
     metricCols.forEach(col => {
         const vals = rows.map(r => parseFloat(r[col])).filter(v => !isNaN(v));
-        if (vals.length) best[col] = Math.max(...vals);
+        if (vals.length) best[col] = higherIsBetter ? Math.max(...vals) : Math.min(...vals);
     });
 
     // Build metric cells for a row
@@ -974,6 +983,15 @@ async function switchBaseFamily(baseFamilyKey) {
     assignModelColors();
     renderSidebar();
     updateDependentFilters(true);
+    // Fall back to a metric that has data in this family (e.g. LLM tps vs CV ips)
+    const hasData = (col) => DATA.some(r =>
+        r[col] !== undefined && r[col] !== null && r[col] !== 0 && r[col] !== "N/A");
+    if (!hasData(filters.metric)) {
+        const familyDefault = config.model_families?.[activeFamilyKey()]?.chart?.default_metric;
+        filters.metric = (familyDefault && hasData(familyDefault))
+            ? familyDefault
+            : (config.metrics.find(m => hasData(m.column))?.column || filters.metric);
+    }
     render();
 }
 

config.json

@@ -9,7 +9,15 @@
     "Llama-3.2-1B": "Llama-3.2-1B-Instruct",
     "Llama-3.2-3B": "Llama-3.2-3B-Instruct",
     "Gemma-3-1B": "gemma-3-1b-it",
-    "Gemma-3-270M": "gemma-3-270m-it"
+    "Gemma-3-270M": "gemma-3-270m-it",
+    "DINOv3-ViT-B16": "dinov3-vitb16-pretrain-lvd1689m",
+    "SAM3": "sam3",
+    "SAM-3D-Body": "sam-3d-body-dinov3",
+    "Parakeet-TDT-0.6B-v3": "parakeet-tdt-0.6b-v3",
+    "Chronos-2": "chronos-2",
+    "MobileViT-Small": "mobilevit-small",
+    "all-MiniLM-L6-v2": "all-MiniLM-L6-v2",
+    "paraphrase-multilingual-MiniLM-L12-v2": "paraphrase-multilingual-MiniLM-L12-v2"
   },
   "filters": [
     {
@@ -29,7 +37,8 @@
         "orin_nano_super": "NVIDIA Jetson Orin Nano Super",
         "agx_orin": "NVIDIA Jetson AGX Orin",
         "agx_thor": "NVIDIA Jetson AGX Thor",
-        "rtx_3500_ada": "NVIDIA RTX 3500 Ada"
+        "rtx_3500_ada": "NVIDIA RTX 3500 Ada",
+        "l4": "NVIDIA L4 GPU"
       }
     }
   ],
@@ -61,6 +70,27 @@
       "short": "E2E(s) ↓",
       "higher_is_better": false,
       "description": "End-to-end latency in seconds (lower is better). Total time from request submission to completion of the full generated response. This reflects real user-perceived latency."
+    },
+    {
+      "column": "ips",
+      "label": "Inferences / sec",
+      "short": "IPS ↑",
+      "higher_is_better": true,
+      "description": "Inferences per second at the stated batch size (higher is better). GPU compute throughput of the TensorRT engine."
+    },
+    {
+      "column": "lat",
+      "label": "Mean Latency (ms)",
+      "short": "LAT(ms) ↓",
+      "higher_is_better": false,
+      "description": "Mean GPU compute latency per inference in milliseconds (lower is better). Host<->device transfers excluded."
+    },
+    {
+      "column": "mem",
+      "label": "Peak Memory (MB)",
+      "short": "MEM(MB) ↓",
+      "higher_is_better": false,
+      "description": "Peak device memory during inference in MB (lower is better)."
     }
   ],
   "display_columns": [
@@ -93,6 +123,16 @@
           "video"
         ]
       }
+    },
+    {
+      "column": "ctx",
+      "label": "CONTEXT",
+      "type": "number",
+      "visible_when": {
+        "type": [
+          "timeseries"
+        ]
+      }
     }
   ],
   "chart": {
@@ -259,6 +299,123 @@
         "rtx_3500_ada": "Measurement setup: vLLM 0.10.2, FlashHead 0.1.7, batch_size=1, 32 input tokens, 128 output tokens generated, 10 warm-up runs, averaged over 100 runs."
       },
       "default_device": "agx_orin"
+    },
+    "DINOv3-ViT-B16": {
+      "data_file": "data/DINOv3-ViT-B16.csv",
+      "accuracy_url": "https://huggingface.co/embedl/dinov3-quantized-tensorrt",
+      "chart": {
+        "default_metric": "ips",
+        "scenarios": []
+      },
+      "default_device": "agx_orin",
+      "experiment_setup": {
+        "l4": "Measurement setup: TensorRT 10.16 Python API mirroring the trtexec protocol (builder optimization level 5, 2 s warmup + 10 s timed, CUDA-event timing, no transfers); stock clocks (datacenter boost). Embedl INT8 vs FP16 baseline of the unmodified model.",
+        "agx_orin": "Measurement setup: trtexec, --builderOptimizationLevel=5, profile flags --useCudaGraph --useSpinWait --noDataTransfers --warmUp=2000 --duration=10; GPU compute time only; Jetson clocks locked (nvpmodel -m 0 + jetson_clocks). Embedl INT8 (--fp16 --int8, calibrated Q/DQ) vs FP16 baseline of the unmodified model."
+      },
+      "accuracy_file": "data/acc-DINOv3-ViT-B16.csv",
+      "accuracy_title": "imagenette k-NN top-1 (%)"
+    },
+    "SAM3": {
+      "data_file": "data/SAM3.csv",
+      "accuracy_url": "https://huggingface.co/embedl/sam3",
+      "chart": {
+        "default_metric": "ips",
+        "scenarios": []
+      },
+      "default_device": "agx_orin",
+      "experiment_setup": {
+        "agx_orin": "Values from the published model-card benchmark card: 'fp16-trt' baseline vs Embedl-optimized 'fp16-int8-trt', batch 1. Latency derived as 1000/FPS.",
+        "agx_thor": "Values from the published model-card benchmark card: 'bf16-torch' baseline vs Embedl-optimized 'fp8-fp16-trt', batch 1. Latency derived as 1000/FPS.",
+        "l4": "Values from the published model-card benchmark card: 'bf16-torch' baseline vs Embedl-optimized 'fp16-int8-trt', batch 1. Latency derived as 1000/FPS."
+      }
+    },
+    "SAM-3D-Body": {
+      "data_file": "data/SAM-3D-Body.csv",
+      "accuracy_url": "https://huggingface.co/embedl/sam-3d-body",
+      "chart": {
+        "default_metric": "ips",
+        "scenarios": []
+      },
+      "default_device": "l4",
+      "experiment_setup": {
+        "l4": "Values from the published model-card benchmark card: 'fp16-trt' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
+      }
+    },
+    "Parakeet-TDT-0.6B-v3": {
+      "data_file": "data/Parakeet-TDT-0.6B-v3.csv",
+      "accuracy_url": "https://huggingface.co/embedl/parakeet-tdt-0.6b-v3-quantized-tensorrt",
+      "chart": {
+        "default_metric": "ips",
+        "scenarios": []
+      },
+      "accuracy_higher_is_better": false,
+      "default_device": "agx_orin",
+      "experiment_setup": {
+        "agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
+      },
+      "accuracy_file": "data/acc-Parakeet-TDT-0.6B-v3.csv",
+      "accuracy_title": "Open ASR Leaderboard WER (%)"
+    },
+    "Chronos-2": {
+      "data_file": "data/Chronos-2.csv",
+      "accuracy_url": "https://huggingface.co/embedl/chronos-2-quantized-trt",
+      "chart": {
+        "default_metric": "ips",
+        "scenarios": [
+          {
+            "label": "Context 512",
+            "match": {
+              "ctx": 512
+            }
+          },
+          {
+            "label": "Context 2048",
+            "match": {
+              "ctx": 2048
+            }
+          }
+        ]
+      },
+      "default_device": "agx_orin",
+      "experiment_setup": {
+        "agx_orin": "Values from the published model-card benchmark card: 'TensorRT FP16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
+      }
+    },
+    "MobileViT-Small": {
+      "data_file": "data/MobileViT-Small.csv",
+      "accuracy_url": "https://huggingface.co/embedl/mobilevit-small-quantized",
+      "chart": {
+        "default_metric": "ips",
+        "scenarios": []
+      },
+      "default_device": "agx_orin",
+      "experiment_setup": {
+        "agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
+      }
+    },
+    "all-MiniLM-L6-v2": {
+      "data_file": "data/all-MiniLM-L6-v2.csv",
+      "accuracy_url": "https://huggingface.co/embedl/all-MiniLM-L6-v2-quantized-trt",
+      "chart": {
+        "default_metric": "ips",
+        "scenarios": []
+      },
+      "default_device": "agx_orin",
+      "experiment_setup": {
+        "agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
+      }
+    },
+    "paraphrase-multilingual-MiniLM-L12-v2": {
+      "data_file": "data/paraphrase-multilingual-MiniLM-L12-v2.csv",
+      "accuracy_url": "https://huggingface.co/embedl/paraphrase-multilingual-MiniLM-L12-v2-quantized-trt",
+      "chart": {
+        "default_metric": "ips",
+        "scenarios": []
+      },
+      "default_device": "agx_orin",
+      "experiment_setup": {
+        "agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
+      }
     }
   },
   "accuracy_title": "Accuracy"

data/Chronos-2.csv

@@ -0,0 +1,5 @@
+model_family,model,type,batch,device,res,ctx,lat,ips,mem
+Chronos-2,amazon/chronos-2,timeseries,1,agx_orin,N/A,2048,4.48,223,
+Chronos-2,embedl/chronos-2-quantized-trt,timeseries,1,agx_orin,N/A,2048,3.48,287,
+Chronos-2,amazon/chronos-2,timeseries,1,agx_orin,N/A,512,2.98,336,
+Chronos-2,embedl/chronos-2-quantized-trt,timeseries,1,agx_orin,N/A,512,2.43,411,

data/DINOv3-ViT-B16.csv

@@ -0,0 +1,5 @@
+model_family,model,type,batch,device,res,ctx,lat,ips,mem
+DINOv3-ViT-B16,facebook/dinov3-vitb16-pretrain-lvd1689m,image,1,agx_orin,224x224,,2.706,369.6,168
+DINOv3-ViT-B16,embedl/dinov3-quantized-tensorrt,image,1,agx_orin,224x224,,2.246,445.3,91.1
+DINOv3-ViT-B16,facebook/dinov3-vitb16-pretrain-lvd1689m,image,1,l4,224x224,,1.504,664.8,
+DINOv3-ViT-B16,embedl/dinov3-quantized-tensorrt,image,1,l4,224x224,,1.127,887.7,

data/MobileViT-Small.csv

@@ -0,0 +1,3 @@
+model_family,model,type,batch,device,res,ctx,lat,ips,mem
+MobileViT-Small,apple/mobilevit-small,image,1,agx_orin,256x256,,1.28,781,
+MobileViT-Small,embedl/mobilevit-small-quantized,image,1,agx_orin,256x256,,1.09,917,

data/Parakeet-TDT-0.6B-v3.csv

@@ -0,0 +1,3 @@
+model_family,model,type,batch,device,res,ctx,lat,ips,mem
+Parakeet-TDT-0.6B-v3,nvidia/parakeet-tdt-0.6b-v3,audio,1,agx_orin,N/A,,33.1,30.2,
+Parakeet-TDT-0.6B-v3,embedl/parakeet-tdt-0.6b-v3-quantized-tensorrt,audio,1,agx_orin,N/A,,33.6,29.8,

data/SAM-3D-Body.csv

@@ -0,0 +1,3 @@
+model_family,model,type,batch,device,res,ctx,lat,ips,mem
+SAM-3D-Body,facebook/sam-3d-body-dinov3,image,1,l4,512x512,,44.6,22.4,
+SAM-3D-Body,embedl/sam-3d-body,image,1,l4,512x512,,28.3,35.3,

data/SAM3.csv

@@ -0,0 +1,7 @@
+model_family,model,type,batch,device,res,ctx,lat,ips,mem
+SAM3,facebook/sam3,image,1,agx_orin,924x924,,769,1.3,
+SAM3,embedl/sam3,image,1,agx_orin,924x924,,455,2.2,
+SAM3,facebook/sam3,image,1,agx_thor,924x924,,137,7.3,
+SAM3,embedl/sam3,image,1,agx_thor,924x924,,90.9,11,
+SAM3,facebook/sam3,image,1,l4,924x924,,137,7.3,
+SAM3,embedl/sam3,image,1,l4,924x924,,104,9.6,

data/acc-DINOv3-ViT-B16.csv

@@ -0,0 +1,3 @@
+Model,knn top1
+facebook/dinov3-vitb16-pretrain-lvd1689m,99.75
+embedl/dinov3-quantized-tensorrt,99.26

data/acc-Parakeet-TDT-0.6B-v3.csv

@@ -0,0 +1,3 @@
+Model,wer
+nvidia/parakeet-tdt-0.6b-v3,6.65
+embedl/parakeet-tdt-0.6b-v3-quantized-tensorrt,6.8

data/all-MiniLM-L6-v2.csv

@@ -0,0 +1,3 @@
+model_family,model,type,batch,device,res,ctx,lat,ips,mem
+all-MiniLM-L6-v2,sentence-transformers/all-MiniLM-L6-v2,text,1,agx_orin,N/A,,0.409,2447,43
+all-MiniLM-L6-v2,embedl/all-MiniLM-L6-v2-quantized-trt,text,1,agx_orin,N/A,,0.383,2611,36

data/paraphrase-multilingual-MiniLM-L12-v2.csv

@@ -0,0 +1,3 @@
+model_family,model,type,batch,device,res,ctx,lat,ips,mem
+paraphrase-multilingual-MiniLM-L12-v2,sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2,text,1,agx_orin,N/A,,0.775,1290,224
+paraphrase-multilingual-MiniLM-L12-v2,embedl/paraphrase-multilingual-MiniLM-L12-v2-quantized-trt,text,1,agx_orin,N/A,,0.73,1369,211