browser/comparison.json

{
  "source_revision": "d439315bd9a11409584e16758bb76a9d75b5bea7",
  "source_encoder_dtype": "torch.bfloat16",
  "scope": "Four English/Chinese text cases for numerical comparison.",
  "cases": [
    {
      "context": "The customer was charged twice for the same order and wants the duplicate payment returned.",
      "question": "Which team should handle this request?",
      "candidates": [
        "Billing",
        "Technical support",
        "Sales"
      ],
      "tokens": 40,
      "reference": [
        0.9212515354156494,
        0.04107579588890076,
        0.037672750651836395
      ],
      "quantized": [
        0.8891163468360901,
        0.05538984015583992,
        0.05549393594264984
      ],
      "quantized_logits": [
        2.8020951747894287,
        0.0262632817029953,
        0.028140777722001076
      ],
      "top1_agrees": true,
      "max_probability_error": 0.032135188579559326,
      "onnx_cpu_seconds": 0.33495458390098065
    },
    {
      "context": "The red box is empty. The blue box contains the key.",
      "question": "Which box contains the key?",
      "candidates": [
        "The red box",
        "The blue box"
      ],
      "tokens": 39,
      "reference": [
        0.020697807893157005,
        0.9793022274971008
      ],
      "quantized": [
        0.03171052783727646,
        0.9682894349098206
      ],
      "quantized_logits": [
        0.6149787306785583,
        4.03386116027832
      ],
      "top1_agrees": true,
      "max_probability_error": 0.011012792587280273,
      "onnx_cpu_seconds": 0.2776173329912126
    },
    {
      "context": "A parcel must arrive by Friday. Express delivery arrives Thursday; standard delivery arrives Monday.",
      "question": "Which delivery option meets the deadline?",
      "candidates": [
        "Express delivery",
        "Standard delivery"
      ],
      "tokens": 40,
      "reference": [
        0.30602750182151794,
        0.6939724683761597
      ],
      "quantized": [
        0.26124119758605957,
        0.7387588024139404
      ],
      "quantized_logits": [
        2.2718594074249268,
        3.311386823654175
      ],
      "top1_agrees": true,
      "max_probability_error": 0.04478633403778076,
      "onnx_cpu_seconds": 0.2724832500098273
    },
    {
      "context": "用户说：订单被重复扣款，希望退回多付的钱。",
      "question": "应该把请求转给哪个团队？",
      "candidates": [
        "账单与退款",
        "技术支持",
        "销售咨询"
      ],
      "tokens": 47,
      "reference": [
        0.9347994327545166,
        0.04668309539556503,
        0.018517442047595978
      ],
      "quantized": [
        0.8895391821861267,
        0.08191443234682083,
        0.028546439483761787
      ],
      "quantized_logits": [
        2.27278733253479,
        -0.11224094033241272,
        -1.166383981704712
      ],
      "top1_agrees": true,
      "max_probability_error": 0.04526025056838989,
      "onnx_cpu_seconds": 0.360311041935347
    }
  ]
}