{
  "chip" : "Apple M5 Pro",
  "elapsed_s" : 127.73304998874664,
  "latency_ms" : {
    "p50" : 5.3043329999999997,
    "p95" : 18.064624999999999
  },
  "lengths" : [
    128,
    256,
    512,
    1024
  ],
  "load_s" : 30.526584982872009,
  "questions" : 3899,
  "suites" : {
    "app.email_spam" : {
      "accuracy" : 0.99250000000000005,
      "buckets" : {
        "L128" : 98,
        "L256" : 117,
        "L512" : 93,
        "L1024" : 92
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 7.062875,
        "p95" : 18.560207999999999
      },
      "max_probability_delta_vs_reference" : 0.024192176759243011,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 9
    },
    "app.guardrails_jailbreak" : {
      "accuracy" : 0.81000000000000005,
      "buckets" : {
        "L128" : 285,
        "L256" : 59,
        "L512" : 53,
        "L1024" : 3
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 4.0982079999999996,
        "p95" : 9.5879580000000004
      },
      "max_probability_delta_vs_reference" : 0.80097097158432007,
      "n" : 400,
      "reference_argmax_agreement" : 0.98999999999999999,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "app.model_routing_domain" : {
      "accuracy" : 0.45363408521303256,
      "buckets" : {
        "L128" : 137,
        "L256" : 261,
        "L512" : 1
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.6524590000000003,
        "p95" : 6.3639999999999999
      },
      "max_probability_delta_vs_reference" : 0.077826619148254395,
      "n" : 399,
      "reference_argmax_agreement" : 0.97493734335839599,
      "reference_compared" : 399,
      "state_truncated" : 0
    },
    "app.moderation_toxicity" : {
      "accuracy" : 0.53500000000000003,
      "buckets" : {
        "L128" : 306,
        "L256" : 54,
        "L512" : 37,
        "L1024" : 3
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 4.0458749999999997,
        "p95" : 10.238166
      },
      "max_probability_delta_vs_reference" : 0.016416072845458984,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "app.phishing" : {
      "accuracy" : 0.99250000000000005,
      "buckets" : {
        "L128" : 76,
        "L256" : 111,
        "L512" : 104,
        "L1024" : 109
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 9.062875,
        "p95" : 19.319541999999998
      },
      "max_probability_delta_vs_reference" : 0.034952796995639801,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 7
    },
    "app.rag_relevance" : {
      "accuracy" : 0.67500000000000004,
      "buckets" : {
        "L128" : 135,
        "L256" : 262,
        "L512" : 3
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.865875,
        "p95" : 9.0795840000000005
      },
      "max_probability_delta_vs_reference" : 0.030605673789978027,
      "n" : 400,
      "reference_argmax_agreement" : 0.99250000000000005,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "app.support_triage" : {
      "accuracy" : 0.53749999999999998,
      "buckets" : {
        "L256" : 377,
        "L512" : 23
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.2802090000000002,
        "p95" : 8.903041
      },
      "max_probability_delta_vs_reference" : 0.052110105752944946,
      "n" : 400,
      "reference_argmax_agreement" : 0.99750000000000005,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "jev.ag_news" : {
      "accuracy" : 0.93500000000000005,
      "buckets" : {
        "L128" : 368,
        "L256" : 32
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 4.5663749999999999,
        "p95" : 11.922083000000001
      },
      "max_probability_delta_vs_reference" : 0.055396914482116699,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "jev.emotion" : {
      "accuracy" : 0.53500000000000003,
      "buckets" : {
        "L128" : 400
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 3.8882919999999999,
        "p95" : 4.2419169999999999
      },
      "max_probability_delta_vs_reference" : 0.086301475763320923,
      "n" : 400,
      "reference_argmax_agreement" : 0.995,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "massive_intent.en" : {
      "accuracy" : 0.65333333333333332,
      "buckets" : {
        "L256" : 300
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.5202499999999999,
        "p95" : 6.7780420000000001
      },
      "max_probability_delta_vs_reference" : 0.14386069774627686,
      "n" : 300,
      "reference_argmax_agreement" : 0.98666666666666669,
      "reference_compared" : 300,
      "state_truncated" : 0
    }
  }
}