Magzimilian Claude Fable 5 commited on
Commit
93c3d46
·
1 Parent(s): fdab5db

Add accuracy tables for five families missing them

Browse files

From each model card: all-MiniLM-L6-v2 (STS-B), MobileViT-Small
(ImageNet), paraphrase-multilingual-MiniLM-L12-v2 (multilingual STS),
SAM3 (SA-Co/Gold), Chronos-2 (WQL ratio, lower-is-better flag). Also
point RF-DETR accuracy_url at the renamed repo.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

config.json CHANGED
@@ -328,7 +328,9 @@
328
  "agx_orin": "Values from the model card's Performance table: FP16 baseline (resized to 924) 763 ms / 1.31 qps vs Embedl-optimized 462 ms / 2.17 qps, batch 1.",
329
  "agx_thor": "Values from the published model-card benchmark card: 'bf16-torch' baseline vs Embedl-optimized 'fp8-fp16-trt', batch 1. Latency derived as 1000/FPS.",
330
  "l4": "Values from the published model-card benchmark card: 'bf16-torch' baseline vs Embedl-optimized 'fp16-int8-trt', batch 1. Latency derived as 1000/FPS."
331
- }
 
 
332
  },
333
  "SAM-3D-Body": {
334
  "data_file": "data/SAM-3D-Body.csv",
@@ -381,7 +383,10 @@
381
  "default_device": "agx_orin",
382
  "experiment_setup": {
383
  "agx_orin": "Values from the published model-card benchmark card: 'TensorRT FP16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
384
- }
 
 
 
385
  },
386
  "MobileViT-Small": {
387
  "data_file": "data/MobileViT-Small.csv",
@@ -393,7 +398,9 @@
393
  "default_device": "agx_orin",
394
  "experiment_setup": {
395
  "agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
396
- }
 
 
397
  },
398
  "all-MiniLM-L6-v2": {
399
  "data_file": "data/all-MiniLM-L6-v2.csv",
@@ -405,7 +412,9 @@
405
  "default_device": "agx_orin",
406
  "experiment_setup": {
407
  "agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
408
- }
 
 
409
  },
410
  "paraphrase-multilingual-MiniLM-L12-v2": {
411
  "data_file": "data/paraphrase-multilingual-MiniLM-L12-v2.csv",
@@ -417,11 +426,13 @@
417
  "default_device": "agx_orin",
418
  "experiment_setup": {
419
  "agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
420
- }
 
 
421
  },
422
  "RF-DETR-Base": {
423
  "data_file": "data/RF-DETR-Base.csv",
424
- "accuracy_url": "https://huggingface.co/embedl/rf-detr-base",
425
  "chart": {
426
  "default_metric": "ips",
427
  "scenarios": []
 
328
  "agx_orin": "Values from the model card's Performance table: FP16 baseline (resized to 924) 763 ms / 1.31 qps vs Embedl-optimized 462 ms / 2.17 qps, batch 1.",
329
  "agx_thor": "Values from the published model-card benchmark card: 'bf16-torch' baseline vs Embedl-optimized 'fp8-fp16-trt', batch 1. Latency derived as 1000/FPS.",
330
  "l4": "Values from the published model-card benchmark card: 'bf16-torch' baseline vs Embedl-optimized 'fp16-int8-trt', batch 1. Latency derived as 1000/FPS."
331
+ },
332
+ "accuracy_file": "data/acc-SAM3.csv",
333
+ "accuracy_title": "SA-Co/Gold"
334
  },
335
  "SAM-3D-Body": {
336
  "data_file": "data/SAM-3D-Body.csv",
 
383
  "default_device": "agx_orin",
384
  "experiment_setup": {
385
  "agx_orin": "Values from the published model-card benchmark card: 'TensorRT FP16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
386
+ },
387
+ "accuracy_file": "data/acc-Chronos-2.csv",
388
+ "accuracy_title": "Forecast WQL / Seasonal-Naive, ctx=2048 (lower is better)",
389
+ "accuracy_higher_is_better": false
390
  },
391
  "MobileViT-Small": {
392
  "data_file": "data/MobileViT-Small.csv",
 
398
  "default_device": "agx_orin",
399
  "experiment_setup": {
400
  "agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
401
+ },
402
+ "accuracy_file": "data/acc-MobileViT-Small.csv",
403
+ "accuracy_title": "ImageNet"
404
  },
405
  "all-MiniLM-L6-v2": {
406
  "data_file": "data/all-MiniLM-L6-v2.csv",
 
412
  "default_device": "agx_orin",
413
  "experiment_setup": {
414
  "agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
415
+ },
416
+ "accuracy_file": "data/acc-all-MiniLM-L6-v2.csv",
417
+ "accuracy_title": "STS-B (Spearman ρ)"
418
  },
419
  "paraphrase-multilingual-MiniLM-L12-v2": {
420
  "data_file": "data/paraphrase-multilingual-MiniLM-L12-v2.csv",
 
426
  "default_device": "agx_orin",
427
  "experiment_setup": {
428
  "agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
429
+ },
430
+ "accuracy_file": "data/acc-paraphrase-multilingual-MiniLM-L12-v2.csv",
431
+ "accuracy_title": "Multilingual STS (Spearman ρ)"
432
  },
433
  "RF-DETR-Base": {
434
  "data_file": "data/RF-DETR-Base.csv",
435
+ "accuracy_url": "https://huggingface.co/embedl/rf-detr-base-quantized-tensorrt",
436
  "chart": {
437
  "default_metric": "ips",
438
  "scenarios": []
data/acc-Chronos-2.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ Model,Geomean WQL / Seasonal-Naive
2
+ amazon/chronos-2,0.549
3
+ embedl/chronos-2-quantized-trt,0.618
data/acc-MobileViT-Small.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ Model,Top-1,Top-5
2
+ apple/mobilevit-small,78.14,94.08
3
+ embedl/mobilevit-small-quantized,74.83,92.28
data/acc-SAM3.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ Model,cgF1,IL_MCC,pos_µF1
2
+ facebook/sam3,55.56,0.823,67.45
3
+ embedl/sam3,53.77,0.809,66.36
data/acc-all-MiniLM-L6-v2.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ Model,Spearman ρ
2
+ sentence-transformers/all-MiniLM-L6-v2,0.8672
3
+ embedl/all-MiniLM-L6-v2-quantized-trt,0.8646
data/acc-paraphrase-multilingual-MiniLM-L12-v2.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ Model,Spearman ρ
2
+ sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2,0.8130
3
+ embedl/paraphrase-multilingual-MiniLM-L12-v2-quantized-trt,0.8008