Upload benchmark_results.json with huggingface_hub

benchmark_results.json

@@ -5,48 +5,48 @@
     {
       "name": "SAM",
       "is_sam": true,
-      "params": "35.9M",
-      "params_raw": 35911302,
-      "size_mb": 137.0,
-      "json_valid": 84.0,
-      "op_correct": 31.5,
-      "exact_match": 8.5,
-      "latency_ms": 691.6,
+      "params": "33.4M",
+      "params_raw": 33400324,
+      "size_mb": 127.4,
+      "json_valid": 88.0,
+      "op_correct": 63.5,
+      "exact_match": 18.0,
+      "latency_ms": 715.4,
       "n_cases": 200,
       "per_tier": [
         {
           "tier": 1,
           "n": 40,
-          "json_valid": 92.5,
-          "op_correct": 75.0,
-          "exact_match": 20.0
+          "json_valid": 100.0,
+          "op_correct": 82.5,
+          "exact_match": 37.5
         },
         {
           "tier": 2,
           "n": 40,
-          "json_valid": 80.0,
-          "op_correct": 32.5,
-          "exact_match": 0.0
+          "json_valid": 100.0,
+          "op_correct": 70.0,
+          "exact_match": 27.5
         },
         {
           "tier": 3,
           "n": 40,
-          "json_valid": 77.5,
-          "op_correct": 50.0,
-          "exact_match": 22.5
+          "json_valid": 100.0,
+          "op_correct": 100.0,
+          "exact_match": 25.0
         },
         {
           "tier": 4,
           "n": 40,
-          "json_valid": 70.0,
+          "json_valid": 75.0,
           "op_correct": 0.0,
           "exact_match": 0.0
         },
         {
           "tier": 5,
           "n": 40,
-          "json_valid": 100.0,
-          "op_correct": 0.0,
+          "json_valid": 65.0,
+          "op_correct": 65.0,
           "exact_match": 0.0
         }
       ],
@@ -56,14 +56,14 @@
           "n": 2,
           "json_valid": 100.0,
           "op_correct": 100.0,
-          "exact_match": 50.0
+          "exact_match": 100.0
         },
         {
           "domain": "DB",
           "n": 27,
-          "json_valid": 77.8,
-          "op_correct": 48.1,
-          "exact_match": 3.7
+          "json_valid": 66.7,
+          "op_correct": 66.7,
+          "exact_match": 11.1
         },
         {
           "domain": "ECOMMERCE",
@@ -76,8 +76,8 @@
           "domain": "FILE",
           "n": 4,
           "json_valid": 100.0,
-          "op_correct": 50.0,
-          "exact_match": 0.0
+          "op_correct": 100.0,
+          "exact_match": 75.0
         },
         {
           "domain": "HOME",
@@ -89,23 +89,23 @@
         {
           "domain": "HTTP",
           "n": 25,
-          "json_valid": 96.0,
-          "op_correct": 60.0,
-          "exact_match": 40.0
+          "json_valid": 100.0,
+          "op_correct": 64.0,
+          "exact_match": 44.0
         },
         {
           "domain": "MQTT",
           "n": 15,
-          "json_valid": 33.3,
-          "op_correct": 33.3,
-          "exact_match": 0.0
+          "json_valid": 100.0,
+          "op_correct": 100.0,
+          "exact_match": 6.7
         },
         {
           "domain": "ROS",
           "n": 46,
-          "json_valid": 87.0,
-          "op_correct": 26.1,
-          "exact_match": 4.3
+          "json_valid": 97.8,
+          "op_correct": 50.0,
+          "exact_match": 10.9
         },
         {
           "domain": "VEHICLE",
@@ -117,9 +117,9 @@
         {
           "domain": "WORKFLOW",
           "n": 49,
-          "json_valid": 81.6,
-          "op_correct": 0.0,
-          "exact_match": 0.0
+          "json_valid": 71.4,
+          "op_correct": 71.4,
+          "exact_match": 16.3
         }
       ]
     },
@@ -129,10 +129,10 @@
       "params": "361.8M",
       "params_raw": 361821120,
       "size_mb": 690.1,
-      "json_valid": 98.5,
+      "json_valid": 89.0,
       "op_correct": 0.0,
       "exact_match": 0.0,
-      "latency_ms": 1027.1,
+      "latency_ms": 2042.9,
       "n_cases": 200,
       "per_tier": [
         {
@@ -159,7 +159,7 @@
         {
           "tier": 4,
           "n": 40,
-          "json_valid": 92.5,
+          "json_valid": 45.0,
           "op_correct": 0.0,
           "exact_match": 0.0
         },
@@ -224,14 +224,14 @@
         {
           "domain": "ROS",
           "n": 46,
-          "json_valid": 93.5,
+          "json_valid": 76.1,
           "op_correct": 0.0,
           "exact_match": 0.0
         },
         {
           "domain": "VEHICLE",
           "n": 17,
-          "json_valid": 100.0,
+          "json_valid": 35.3,
           "op_correct": 0.0,
           "exact_match": 0.0
         },
@@ -250,10 +250,10 @@
       "params": "494.0M",
       "params_raw": 494032768,
       "size_mb": 942.3,
-      "json_valid": 71.5,
-      "op_correct": 28.5,
+      "json_valid": 70.5,
+      "op_correct": 30.0,
       "exact_match": 2.5,
-      "latency_ms": 2108.7,
+      "latency_ms": 1937.2,
       "n_cases": 200,
       "per_tier": [
         {
@@ -280,8 +280,8 @@
         {
           "tier": 4,
           "n": 40,
-          "json_valid": 100.0,
-          "op_correct": 20.0,
+          "json_valid": 95.0,
+          "op_correct": 27.5,
           "exact_match": 0.0
         },
         {
@@ -303,7 +303,7 @@
         {
           "domain": "DB",
           "n": 27,
-          "json_valid": 51.9,
+          "json_valid": 44.4,
           "op_correct": 7.4,
           "exact_match": 0.0
         },
@@ -353,7 +353,7 @@
           "domain": "VEHICLE",
           "n": 17,
           "json_valid": 82.4,
-          "op_correct": 64.7,
+          "op_correct": 82.4,
           "exact_match": 17.6
         },
         {
@@ -371,10 +371,10 @@
       "params": "1100.0M",
       "params_raw": 1100048384,
       "size_mb": 2098.2,
-      "json_valid": 93.5,
-      "op_correct": 57.0,
+      "json_valid": 92.0,
+      "op_correct": 55.5,
       "exact_match": 9.0,
-      "latency_ms": 4484.0,
+      "latency_ms": 4199.3,
       "n_cases": 200,
       "per_tier": [
         {
@@ -388,7 +388,7 @@
           "tier": 2,
           "n": 40,
           "json_valid": 90.0,
-          "op_correct": 90.0,
+          "op_correct": 82.5,
           "exact_match": 12.5
         },
         {
@@ -401,7 +401,7 @@
         {
           "tier": 4,
           "n": 40,
-          "json_valid": 77.5,
+          "json_valid": 70.0,
           "op_correct": 22.5,
           "exact_match": 0.0
         },
@@ -466,14 +466,14 @@
         {
           "domain": "ROS",
           "n": 46,
-          "json_valid": 80.4,
-          "op_correct": 54.3,
+          "json_valid": 76.1,
+          "op_correct": 47.8,
           "exact_match": 17.4
         },
         {
           "domain": "VEHICLE",
           "n": 17,
-          "json_valid": 100.0,
+          "json_valid": 94.1,
           "op_correct": 35.3,
           "exact_match": 23.5
         },
@@ -505,9 +505,9 @@
       "params_raw": 1543714304,
       "size_mb": 2944.4,
       "json_valid": 100.0,
-      "op_correct": 75.0,
-      "exact_match": 9.5,
-      "latency_ms": 1976.9,
+      "op_correct": 78.5,
+      "exact_match": 9.0,
+      "latency_ms": 1852.4,
       "n_cases": 200,
       "per_tier": [
         {
@@ -521,8 +521,8 @@
           "tier": 2,
           "n": 40,
           "json_valid": 100.0,
-          "op_correct": 90.0,
-          "exact_match": 15.0
+          "op_correct": 85.0,
+          "exact_match": 12.5
         },
         {
           "tier": 3,
@@ -535,7 +535,7 @@
           "tier": 4,
           "n": 40,
           "json_valid": 100.0,
-          "op_correct": 50.0,
+          "op_correct": 72.5,
           "exact_match": 0.0
         },
         {
@@ -558,7 +558,7 @@
           "domain": "DB",
           "n": 27,
           "json_valid": 100.0,
-          "op_correct": 66.7,
+          "op_correct": 100.0,
           "exact_match": 18.5
         },
         {
@@ -600,8 +600,8 @@
           "domain": "ROS",
           "n": 46,
           "json_valid": 100.0,
-          "op_correct": 76.1,
-          "exact_match": 21.7
+          "op_correct": 71.7,
+          "exact_match": 19.6
         },
         {
           "domain": "VEHICLE",
@@ -625,10 +625,10 @@
       "params": "1711.4M",
       "params_raw": 1711376384,
       "size_mb": 3264.2,
-      "json_valid": 85.0,
-      "op_correct": 47.0,
+      "json_valid": 89.0,
+      "op_correct": 38.5,
       "exact_match": 2.5,
-      "latency_ms": 1884.5,
+      "latency_ms": 1694.9,
       "n_cases": 200,
       "per_tier": [
         {
@@ -641,7 +641,7 @@
         {
           "tier": 2,
           "n": 40,
-          "json_valid": 72.5,
+          "json_valid": 80.0,
           "op_correct": 62.5,
           "exact_match": 0.0
         },
@@ -655,15 +655,15 @@
         {
           "tier": 4,
           "n": 40,
-          "json_valid": 60.0,
-          "op_correct": 2.5,
+          "json_valid": 72.5,
+          "op_correct": 0.0,
           "exact_match": 0.0
         },
         {
           "tier": 5,
           "n": 40,
           "json_valid": 100.0,
-          "op_correct": 40.0,
+          "op_correct": 0.0,
           "exact_match": 0.0
         }
       ],
@@ -720,22 +720,22 @@
         {
           "domain": "ROS",
           "n": 46,
-          "json_valid": 89.1,
+          "json_valid": 95.7,
           "op_correct": 47.8,
           "exact_match": 4.3
         },
         {
           "domain": "VEHICLE",
           "n": 17,
-          "json_valid": 58.8,
-          "op_correct": 29.4,
+          "json_valid": 88.2,
+          "op_correct": 23.5,
           "exact_match": 17.6
         },
         {
           "domain": "WORKFLOW",
           "n": 49,
           "json_valid": 83.7,
-          "op_correct": 34.7,
+          "op_correct": 2.0,
           "exact_match": 0.0
         }
       ]
@@ -759,9 +759,9 @@
       "params_raw": 3085938688,
       "size_mb": 5886.0,
       "json_valid": 100.0,
-      "op_correct": 81.5,
-      "exact_match": 17.5,
-      "latency_ms": 2716.7,
+      "op_correct": 82.5,
+      "exact_match": 16.0,
+      "latency_ms": 2061.1,
       "n_cases": 200,
       "per_tier": [
         {
@@ -776,7 +776,7 @@
           "n": 40,
           "json_valid": 100.0,
           "op_correct": 100.0,
-          "exact_match": 20.0
+          "exact_match": 12.5
         },
         {
           "tier": 3,
@@ -789,7 +789,7 @@
           "tier": 4,
           "n": 40,
           "json_valid": 100.0,
-          "op_correct": 70.0,
+          "op_correct": 75.0,
           "exact_match": 0.0
         },
         {
@@ -812,7 +812,7 @@
           "domain": "DB",
           "n": 27,
           "json_valid": 100.0,
-          "op_correct": 66.7,
+          "op_correct": 100.0,
           "exact_match": 18.5
         },
         {
@@ -854,8 +854,8 @@
           "domain": "ROS",
           "n": 46,
           "json_valid": 100.0,
-          "op_correct": 93.5,
-          "exact_match": 28.3
+          "op_correct": 78.3,
+          "exact_match": 21.7
         },
         {
           "domain": "VEHICLE",
@@ -916,9 +916,9 @@
       "params_raw": 270000000,
       "size_mb": 241.4,
       "json_valid": 92.0,
-      "op_correct": 2.5,
+      "op_correct": 2.0,
       "exact_match": 0.0,
-      "latency_ms": 6871.5,
+      "latency_ms": 6322.5,
       "n_cases": 200,
       "per_tier": [
         {
@@ -932,7 +932,7 @@
           "tier": 2,
           "n": 40,
           "json_valid": 97.5,
-          "op_correct": 7.5,
+          "op_correct": 5.0,
           "exact_match": 0.0
         },
         {
@@ -968,7 +968,7 @@
         {
           "domain": "DB",
           "n": 27,
-          "json_valid": 85.2,
+          "json_valid": 88.9,
           "op_correct": 0.0,
           "exact_match": 0.0
         },
@@ -1010,8 +1010,8 @@
         {
           "domain": "ROS",
           "n": 46,
-          "json_valid": 89.1,
-          "op_correct": 2.2,
+          "json_valid": 87.0,
+          "op_correct": 0.0,
           "exact_match": 0.0
         },
         {
@@ -1048,10 +1048,10 @@
       "params": "8000.0M",
       "params_raw": 8000000000,
       "size_mb": 4794.9,
-      "json_valid": 98.5,
-      "op_correct": 79.5,
+      "json_valid": 98.0,
+      "op_correct": 79.0,
       "exact_match": 11.5,
-      "latency_ms": 37642.4,
+      "latency_ms": 31737.9,
       "n_cases": 200,
       "per_tier": [
         {
@@ -1078,8 +1078,8 @@
         {
           "tier": 4,
           "n": 40,
-          "json_valid": 97.5,
-          "op_correct": 95.0,
+          "json_valid": 95.0,
+          "op_correct": 92.5,
           "exact_match": 0.0
         },
         {
@@ -1143,8 +1143,8 @@
         {
           "domain": "ROS",
           "n": 46,
-          "json_valid": 97.8,
-          "op_correct": 71.7,
+          "json_valid": 95.7,
+          "op_correct": 69.6,
           "exact_match": 10.9
         },
         {
@@ -1168,31 +1168,31 @@
     {
       "tier": 1,
       "n": 40,
-      "json_valid": 92.5,
-      "op_correct": 75.0,
-      "exact_match": 20.0,
+      "json_valid": 100.0,
+      "op_correct": 82.5,
+      "exact_match": 37.5,
       "description": "Atomic \u2014 single op"
     },
     {
       "tier": 2,
       "n": 40,
-      "json_valid": 80.0,
-      "op_correct": 32.5,
-      "exact_match": 0.0,
+      "json_valid": 100.0,
+      "op_correct": 70.0,
+      "exact_match": 27.5,
       "description": "Compound \u2014 2-3 ops same domain"
     },
     {
       "tier": 3,
       "n": 40,
-      "json_valid": 77.5,
-      "op_correct": 50.0,
-      "exact_match": 22.5,
+      "json_valid": 100.0,
+      "op_correct": 100.0,
+      "exact_match": 25.0,
       "description": "Noisy \u2014 typos / casing / fillers"
     },
     {
       "tier": 4,
       "n": 40,
-      "json_valid": 70.0,
+      "json_valid": 75.0,
       "op_correct": 0.0,
       "exact_match": 0.0,
       "description": "Long chain \u2014 4+ ops"
@@ -1200,8 +1200,8 @@
     {
       "tier": 5,
       "n": 40,
-      "json_valid": 100.0,
-      "op_correct": 0.0,
+      "json_valid": 65.0,
+      "op_correct": 65.0,
       "exact_match": 0.0,
       "description": "Cross-domain workflow"
     }
@@ -1212,14 +1212,14 @@
       "n": 2,
       "json_valid": 100.0,
       "op_correct": 100.0,
-      "exact_match": 50.0
+      "exact_match": 100.0
     },
     {
       "domain": "DB",
       "n": 27,
-      "json_valid": 77.8,
-      "op_correct": 48.1,
-      "exact_match": 3.7
+      "json_valid": 66.7,
+      "op_correct": 66.7,
+      "exact_match": 11.1
     },
     {
       "domain": "ECOMMERCE",
@@ -1232,8 +1232,8 @@
       "domain": "FILE",
       "n": 4,
       "json_valid": 100.0,
-      "op_correct": 50.0,
-      "exact_match": 0.0
+      "op_correct": 100.0,
+      "exact_match": 75.0
     },
     {
       "domain": "HOME",
@@ -1245,23 +1245,23 @@
     {
       "domain": "HTTP",
       "n": 25,
-      "json_valid": 96.0,
-      "op_correct": 60.0,
-      "exact_match": 40.0
+      "json_valid": 100.0,
+      "op_correct": 64.0,
+      "exact_match": 44.0
     },
     {
       "domain": "MQTT",
       "n": 15,
-      "json_valid": 33.3,
-      "op_correct": 33.3,
-      "exact_match": 0.0
+      "json_valid": 100.0,
+      "op_correct": 100.0,
+      "exact_match": 6.7
     },
     {
       "domain": "ROS",
       "n": 46,
-      "json_valid": 87.0,
-      "op_correct": 26.1,
-      "exact_match": 4.3
+      "json_valid": 97.8,
+      "op_correct": 50.0,
+      "exact_match": 10.9
     },
     {
       "domain": "VEHICLE",
@@ -1273,9 +1273,9 @@
     {
       "domain": "WORKFLOW",
       "n": 49,
-      "json_valid": 81.6,
-      "op_correct": 0.0,
-      "exact_match": 0.0
+      "json_valid": 71.4,
+      "op_correct": 71.4,
+      "exact_match": 16.3
     }
   ],
   "tier_distribution": {