{
  "chip" : "Apple M5 Pro",
  "elapsed_s" : 23.682847023010254,
  "latency_ms" : {
    "p50" : 5.3144580000000001,
    "p95" : 18.081707999999999
  },
  "lengths" : [
    128,
    256,
    512,
    1024
  ],
  "load_s" : 2.8048709630966187,
  "questions" : 3899,
  "suites" : {
    "app.email_spam" : {
      "accuracy" : 0.99250000000000005,
      "buckets" : {
        "L128" : 98,
        "L256" : 117,
        "L512" : 93,
        "L1024" : 92
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 6.297625,
        "p95" : 18.979500000000002
      },
      "max_probability_delta_vs_reference" : 0.0070431232452392578,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 9
    },
    "app.guardrails_jailbreak" : {
      "accuracy" : 0.8075,
      "buckets" : {
        "L128" : 285,
        "L256" : 59,
        "L512" : 53,
        "L1024" : 3
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 4.1935419999999999,
        "p95" : 9.8876670000000004
      },
      "max_probability_delta_vs_reference" : 0.01729881763458252,
      "n" : 400,
      "reference_argmax_agreement" : 0.99750000000000005,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "app.model_routing_domain" : {
      "accuracy" : 0.44110275689223055,
      "buckets" : {
        "L128" : 137,
        "L256" : 261,
        "L512" : 1
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.391,
        "p95" : 6.2202919999999997
      },
      "max_probability_delta_vs_reference" : 0.014412820339202881,
      "n" : 399,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 399,
      "state_truncated" : 0
    },
    "app.moderation_toxicity" : {
      "accuracy" : 0.53500000000000003,
      "buckets" : {
        "L128" : 306,
        "L256" : 54,
        "L512" : 37,
        "L1024" : 3
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 3.8999999999999999,
        "p95" : 9.4122920000000008
      },
      "max_probability_delta_vs_reference" : 0.0036969184875488281,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "app.phishing" : {
      "accuracy" : 0.99250000000000005,
      "buckets" : {
        "L128" : 76,
        "L256" : 111,
        "L512" : 104,
        "L1024" : 109
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 8.9892920000000007,
        "p95" : 18.856666000000001
      },
      "max_probability_delta_vs_reference" : 0.003923460841178894,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 7
    },
    "app.rag_relevance" : {
      "accuracy" : 0.67249999999999999,
      "buckets" : {
        "L128" : 135,
        "L256" : 262,
        "L512" : 3
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.4204169999999996,
        "p95" : 6.3289999999999997
      },
      "max_probability_delta_vs_reference" : 0.0088013410568237305,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "app.support_triage" : {
      "accuracy" : 0.54249999999999998,
      "buckets" : {
        "L256" : 377,
        "L512" : 23
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.5369999999999999,
        "p95" : 9.0276250000000005
      },
      "max_probability_delta_vs_reference" : 0.0048961937427520752,
      "n" : 400,
      "reference_argmax_agreement" : 0.99750000000000005,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "jev.ag_news" : {
      "accuracy" : 0.93500000000000005,
      "buckets" : {
        "L128" : 368,
        "L256" : 32
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 4.6144170000000004,
        "p95" : 7.2706249999999999
      },
      "max_probability_delta_vs_reference" : 0.010490596294403076,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "jev.emotion" : {
      "accuracy" : 0.53749999999999998,
      "buckets" : {
        "L128" : 400
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 4.0142920000000002,
        "p95" : 4.3915829999999998
      },
      "max_probability_delta_vs_reference" : 0.020582765340805054,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "massive_intent.en" : {
      "accuracy" : 0.65666666666666662,
      "buckets" : {
        "L256" : 300
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.3079169999999998,
        "p95" : 6.1710409999999998
      },
      "max_probability_delta_vs_reference" : 0.019814342260360718,
      "n" : 300,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 300,
      "state_truncated" : 0
    }
  }
}