Commit ·
93c3d46
1
Parent(s): fdab5db
Add accuracy tables for five families missing them
Browse filesFrom each model card: all-MiniLM-L6-v2 (STS-B), MobileViT-Small
(ImageNet), paraphrase-multilingual-MiniLM-L12-v2 (multilingual STS),
SAM3 (SA-Co/Gold), Chronos-2 (WQL ratio, lower-is-better flag). Also
point RF-DETR accuracy_url at the renamed repo.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
- config.json +17 -6
- data/acc-Chronos-2.csv +3 -0
- data/acc-MobileViT-Small.csv +3 -0
- data/acc-SAM3.csv +3 -0
- data/acc-all-MiniLM-L6-v2.csv +3 -0
- data/acc-paraphrase-multilingual-MiniLM-L12-v2.csv +3 -0
config.json
CHANGED
|
@@ -328,7 +328,9 @@
|
|
| 328 |
"agx_orin": "Values from the model card's Performance table: FP16 baseline (resized to 924) 763 ms / 1.31 qps vs Embedl-optimized 462 ms / 2.17 qps, batch 1.",
|
| 329 |
"agx_thor": "Values from the published model-card benchmark card: 'bf16-torch' baseline vs Embedl-optimized 'fp8-fp16-trt', batch 1. Latency derived as 1000/FPS.",
|
| 330 |
"l4": "Values from the published model-card benchmark card: 'bf16-torch' baseline vs Embedl-optimized 'fp16-int8-trt', batch 1. Latency derived as 1000/FPS."
|
| 331 |
-
}
|
|
|
|
|
|
|
| 332 |
},
|
| 333 |
"SAM-3D-Body": {
|
| 334 |
"data_file": "data/SAM-3D-Body.csv",
|
|
@@ -381,7 +383,10 @@
|
|
| 381 |
"default_device": "agx_orin",
|
| 382 |
"experiment_setup": {
|
| 383 |
"agx_orin": "Values from the published model-card benchmark card: 'TensorRT FP16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
|
| 384 |
-
}
|
|
|
|
|
|
|
|
|
|
| 385 |
},
|
| 386 |
"MobileViT-Small": {
|
| 387 |
"data_file": "data/MobileViT-Small.csv",
|
|
@@ -393,7 +398,9 @@
|
|
| 393 |
"default_device": "agx_orin",
|
| 394 |
"experiment_setup": {
|
| 395 |
"agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
|
| 396 |
-
}
|
|
|
|
|
|
|
| 397 |
},
|
| 398 |
"all-MiniLM-L6-v2": {
|
| 399 |
"data_file": "data/all-MiniLM-L6-v2.csv",
|
|
@@ -405,7 +412,9 @@
|
|
| 405 |
"default_device": "agx_orin",
|
| 406 |
"experiment_setup": {
|
| 407 |
"agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
|
| 408 |
-
}
|
|
|
|
|
|
|
| 409 |
},
|
| 410 |
"paraphrase-multilingual-MiniLM-L12-v2": {
|
| 411 |
"data_file": "data/paraphrase-multilingual-MiniLM-L12-v2.csv",
|
|
@@ -417,11 +426,13 @@
|
|
| 417 |
"default_device": "agx_orin",
|
| 418 |
"experiment_setup": {
|
| 419 |
"agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
|
| 420 |
-
}
|
|
|
|
|
|
|
| 421 |
},
|
| 422 |
"RF-DETR-Base": {
|
| 423 |
"data_file": "data/RF-DETR-Base.csv",
|
| 424 |
-
"accuracy_url": "https://huggingface.co/embedl/rf-detr-base",
|
| 425 |
"chart": {
|
| 426 |
"default_metric": "ips",
|
| 427 |
"scenarios": []
|
|
|
|
| 328 |
"agx_orin": "Values from the model card's Performance table: FP16 baseline (resized to 924) 763 ms / 1.31 qps vs Embedl-optimized 462 ms / 2.17 qps, batch 1.",
|
| 329 |
"agx_thor": "Values from the published model-card benchmark card: 'bf16-torch' baseline vs Embedl-optimized 'fp8-fp16-trt', batch 1. Latency derived as 1000/FPS.",
|
| 330 |
"l4": "Values from the published model-card benchmark card: 'bf16-torch' baseline vs Embedl-optimized 'fp16-int8-trt', batch 1. Latency derived as 1000/FPS."
|
| 331 |
+
},
|
| 332 |
+
"accuracy_file": "data/acc-SAM3.csv",
|
| 333 |
+
"accuracy_title": "SA-Co/Gold"
|
| 334 |
},
|
| 335 |
"SAM-3D-Body": {
|
| 336 |
"data_file": "data/SAM-3D-Body.csv",
|
|
|
|
| 383 |
"default_device": "agx_orin",
|
| 384 |
"experiment_setup": {
|
| 385 |
"agx_orin": "Values from the published model-card benchmark card: 'TensorRT FP16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
|
| 386 |
+
},
|
| 387 |
+
"accuracy_file": "data/acc-Chronos-2.csv",
|
| 388 |
+
"accuracy_title": "Forecast WQL / Seasonal-Naive, ctx=2048 (lower is better)",
|
| 389 |
+
"accuracy_higher_is_better": false
|
| 390 |
},
|
| 391 |
"MobileViT-Small": {
|
| 392 |
"data_file": "data/MobileViT-Small.csv",
|
|
|
|
| 398 |
"default_device": "agx_orin",
|
| 399 |
"experiment_setup": {
|
| 400 |
"agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
|
| 401 |
+
},
|
| 402 |
+
"accuracy_file": "data/acc-MobileViT-Small.csv",
|
| 403 |
+
"accuracy_title": "ImageNet"
|
| 404 |
},
|
| 405 |
"all-MiniLM-L6-v2": {
|
| 406 |
"data_file": "data/all-MiniLM-L6-v2.csv",
|
|
|
|
| 412 |
"default_device": "agx_orin",
|
| 413 |
"experiment_setup": {
|
| 414 |
"agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
|
| 415 |
+
},
|
| 416 |
+
"accuracy_file": "data/acc-all-MiniLM-L6-v2.csv",
|
| 417 |
+
"accuracy_title": "STS-B (Spearman ρ)"
|
| 418 |
},
|
| 419 |
"paraphrase-multilingual-MiniLM-L12-v2": {
|
| 420 |
"data_file": "data/paraphrase-multilingual-MiniLM-L12-v2.csv",
|
|
|
|
| 426 |
"default_device": "agx_orin",
|
| 427 |
"experiment_setup": {
|
| 428 |
"agx_orin": "Values from the published model-card benchmark card: 'trtexec --fp16' baseline vs Embedl-optimized 'embedl int8', batch 1. Latency derived as 1000/FPS."
|
| 429 |
+
},
|
| 430 |
+
"accuracy_file": "data/acc-paraphrase-multilingual-MiniLM-L12-v2.csv",
|
| 431 |
+
"accuracy_title": "Multilingual STS (Spearman ρ)"
|
| 432 |
},
|
| 433 |
"RF-DETR-Base": {
|
| 434 |
"data_file": "data/RF-DETR-Base.csv",
|
| 435 |
+
"accuracy_url": "https://huggingface.co/embedl/rf-detr-base-quantized-tensorrt",
|
| 436 |
"chart": {
|
| 437 |
"default_metric": "ips",
|
| 438 |
"scenarios": []
|
data/acc-Chronos-2.csv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Model,Geomean WQL / Seasonal-Naive
|
| 2 |
+
amazon/chronos-2,0.549
|
| 3 |
+
embedl/chronos-2-quantized-trt,0.618
|
data/acc-MobileViT-Small.csv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Model,Top-1,Top-5
|
| 2 |
+
apple/mobilevit-small,78.14,94.08
|
| 3 |
+
embedl/mobilevit-small-quantized,74.83,92.28
|
data/acc-SAM3.csv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Model,cgF1,IL_MCC,pos_µF1
|
| 2 |
+
facebook/sam3,55.56,0.823,67.45
|
| 3 |
+
embedl/sam3,53.77,0.809,66.36
|
data/acc-all-MiniLM-L6-v2.csv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Model,Spearman ρ
|
| 2 |
+
sentence-transformers/all-MiniLM-L6-v2,0.8672
|
| 3 |
+
embedl/all-MiniLM-L6-v2-quantized-trt,0.8646
|
data/acc-paraphrase-multilingual-MiniLM-L12-v2.csv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Model,Spearman ρ
|
| 2 |
+
sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2,0.8130
|
| 3 |
+
embedl/paraphrase-multilingual-MiniLM-L12-v2-quantized-trt,0.8008
|