| { |
| "results": [ |
| { |
| "model": "laya", |
| "revision": "c5d78730f3493e4fe16d61507ef4b78eef7318cf", |
| "dtype": "float32", |
| "argmax_agreements": 63, |
| "questions": 63, |
| "probability_max_abs_error": 5.170394474396378e-06, |
| "probability_tolerance": 0.0001, |
| "cases": [ |
| { |
| "case": "email", |
| "questions": 3, |
| "sequence_length": 93, |
| "input_sha256": "97703a282ecbb252db9f7e3121ad77e9d1a6671656cf35e62e60f0efeffd032c", |
| "tokens_equal": true, |
| "logits_max_abs_error": 9.387731552124023e-06, |
| "action_logits_max_abs_error": 0.0126953125, |
| "probability_max_abs_error": 1.9275840926757937e-06, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "en", |
| "questions": 3, |
| "sequence_length": 65, |
| "input_sha256": "f89895466f2fafda31ca556a46d3823764e73156772693c88cc148edcc6a39eb", |
| "tokens_equal": true, |
| "logits_max_abs_error": 3.814697265625e-06, |
| "action_logits_max_abs_error": 0.00244140625, |
| "probability_max_abs_error": 5.839881763725074e-07, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "zh", |
| "questions": 3, |
| "sequence_length": 73, |
| "input_sha256": "0872ab22fd69ba60ca320db76dc09d90a9de1f4ebd4aaeafd46169a763f20e3f", |
| "tokens_equal": true, |
| "logits_max_abs_error": 1.0251998901367188e-05, |
| "action_logits_max_abs_error": 0.001953125, |
| "probability_max_abs_error": 1.5244581486606457e-06, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "de", |
| "questions": 3, |
| "sequence_length": 76, |
| "input_sha256": "88c970581ad10b6bd1aa82afaaef93513e456e68cf77e7e3cad98919de3ff1a5", |
| "tokens_equal": true, |
| "logits_max_abs_error": 6.67572021484375e-06, |
| "action_logits_max_abs_error": 0.00146484375, |
| "probability_max_abs_error": 2.909347931456807e-07, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "fr", |
| "questions": 3, |
| "sequence_length": 81, |
| "input_sha256": "00d71ea5bfd209fbbeffad3decdfacd9a51a4a97c07b7ab8969af5aa8bd51e43", |
| "tokens_equal": true, |
| "logits_max_abs_error": 6.079673767089844e-06, |
| "action_logits_max_abs_error": 0.014404296875, |
| "probability_max_abs_error": 7.556087676663381e-07, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "es", |
| "questions": 3, |
| "sequence_length": 75, |
| "input_sha256": "3d429a6d7f31bc59256e88c56a1d7a8ae40a05c6277fc6d22341bdc2203d93b6", |
| "tokens_equal": true, |
| "logits_max_abs_error": 1.3828277587890625e-05, |
| "action_logits_max_abs_error": 0.00830078125, |
| "probability_max_abs_error": 1.2019799846552104e-06, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "hi", |
| "questions": 3, |
| "sequence_length": 119, |
| "input_sha256": "23f2a78bed3a972ae02a7d3dc26e5b0eec62aee4230ff0c5a1388d7f044dd121", |
| "tokens_equal": true, |
| "logits_max_abs_error": 3.93986701965332e-05, |
| "action_logits_max_abs_error": 0.0205078125, |
| "probability_max_abs_error": 5.170394474396378e-06, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "ja", |
| "questions": 3, |
| "sequence_length": 75, |
| "input_sha256": "e73ae1efad1e50525b04f6a8f8004ccc15478f73ac3618fbe752e25c1c6c150b", |
| "tokens_equal": true, |
| "logits_max_abs_error": 1.621246337890625e-05, |
| "action_logits_max_abs_error": 0.0029296875, |
| "probability_max_abs_error": 1.8448868603915258e-06, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "ru", |
| "questions": 3, |
| "sequence_length": 85, |
| "input_sha256": "bac6676f433fdbb4aa2b71dda957299b7e5a9efa2c2b526f000f51d8d4be3a08", |
| "tokens_equal": true, |
| "logits_max_abs_error": 2.2411346435546875e-05, |
| "action_logits_max_abs_error": 0.00341796875, |
| "probability_max_abs_error": 1.3785159388851609e-06, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "empty_state", |
| "questions": 3, |
| "sequence_length": 46, |
| "input_sha256": "85f8a747931b65f225679d74c32afc65b23c679549560503921a969453d2c4e1", |
| "tokens_equal": true, |
| "logits_max_abs_error": 1.1205673217773438e-05, |
| "action_logits_max_abs_error": 0.0009765625, |
| "probability_max_abs_error": 8.773777885884293e-07, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "long", |
| "questions": 3, |
| "sequence_length": 512, |
| "input_sha256": "029a742cb223bbb34add55cb46a0d266345a2e448f314ced18033f3710a9c8c7", |
| "tokens_equal": true, |
| "logits_max_abs_error": 6.556510925292969e-06, |
| "action_logits_max_abs_error": 0.00048828125, |
| "probability_max_abs_error": 1.7214299026235835e-06, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "conversation", |
| "questions": 3, |
| "sequence_length": 72, |
| "input_sha256": "a429d0393a0bd2af5d960fa45ab286d6741b89969693fa852560a6c87309a4d0", |
| "tokens_equal": true, |
| "logits_max_abs_error": 8.344650268554688e-06, |
| "action_logits_max_abs_error": 0.00244140625, |
| "probability_max_abs_error": 5.85747966019845e-07, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "mask_literals", |
| "questions": 3, |
| "sequence_length": 55, |
| "input_sha256": "f89667d91cd59690e55b05e6af9368ea1e27ef42829525a97d4cd3cace2f1989", |
| "tokens_equal": true, |
| "logits_max_abs_error": 1.1444091796875e-05, |
| "action_logits_max_abs_error": 0.0009765625, |
| "probability_max_abs_error": 9.02355455134618e-07, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "many_questions", |
| "questions": 20, |
| "sequence_length": 93, |
| "input_sha256": "47942741e9ed5fabc4c49464b633b43608c61d04c9bc3f23fea5c0ae4da23ae3", |
| "tokens_equal": true, |
| "logits_max_abs_error": 7.152557373046875e-06, |
| "action_logits_max_abs_error": 0.01220703125, |
| "probability_max_abs_error": 4.862327878663031e-07, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 20, |
| "public_result_equal": true |
| }, |
| { |
| "case": "structured", |
| "questions": 3, |
| "sequence_length": 78, |
| "input_sha256": "8ba1aac3447918529ea99ae8c34146ebe16db359e7787412e6dda90c34f9b8b2", |
| "tokens_equal": true, |
| "logits_max_abs_error": 1.1727213859558105e-05, |
| "action_logits_max_abs_error": 0.00146484375, |
| "probability_max_abs_error": 9.181504970756649e-07, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": true |
| }, |
| { |
| "case": "twenty_options", |
| "questions": 1, |
| "sequence_length": 137, |
| "input_sha256": "f20fe7490e99ad43d4d1c5a9657f095968b904c9a60e8b3e02fce0544126f838", |
| "tokens_equal": true, |
| "logits_max_abs_error": 2.0503997802734375e-05, |
| "action_logits_max_abs_error": 0.0068359375, |
| "probability_max_abs_error": 6.2494302366671145e-34, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 1, |
| "public_result_equal": true |
| } |
| ], |
| "stability": { |
| "calls": 100, |
| "elapsed_seconds": 3.9830371670013847, |
| "finite": true, |
| "deterministic": true, |
| "active_bytes_before": 1685193880, |
| "active_bytes_after": 1685193880, |
| "active_growth_bytes": 0, |
| "peak_bytes": 3296604064 |
| } |
| }, |
| { |
| "model": "laya", |
| "revision": "c5d78730f3493e4fe16d61507ef4b78eef7318cf", |
| "dtype": "float16", |
| "argmax_agreements": 63, |
| "questions": 63, |
| "probability_max_abs_error": 0.005444271459395356, |
| "probability_tolerance": 0.02, |
| "cases": [ |
| { |
| "case": "email", |
| "questions": 3, |
| "sequence_length": 93, |
| "input_sha256": "97703a282ecbb252db9f7e3121ad77e9d1a6671656cf35e62e60f0efeffd032c", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.0036835670471191406, |
| "action_logits_max_abs_error": 1.83251953125, |
| "probability_max_abs_error": 0.0008600417756422507, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "en", |
| "questions": 3, |
| "sequence_length": 65, |
| "input_sha256": "f89895466f2fafda31ca556a46d3823764e73156772693c88cc148edcc6a39eb", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.005011081695556641, |
| "action_logits_max_abs_error": 3.49658203125, |
| "probability_max_abs_error": 0.0003116258116069026, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "zh", |
| "questions": 3, |
| "sequence_length": 73, |
| "input_sha256": "0872ab22fd69ba60ca320db76dc09d90a9de1f4ebd4aaeafd46169a763f20e3f", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.04189777374267578, |
| "action_logits_max_abs_error": 17.02685546875, |
| "probability_max_abs_error": 0.0013265283093753522, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "de", |
| "questions": 3, |
| "sequence_length": 76, |
| "input_sha256": "88c970581ad10b6bd1aa82afaaef93513e456e68cf77e7e3cad98919de3ff1a5", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.005423009395599365, |
| "action_logits_max_abs_error": 0.80078125, |
| "probability_max_abs_error": 0.0002973207056953253, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "fr", |
| "questions": 3, |
| "sequence_length": 81, |
| "input_sha256": "00d71ea5bfd209fbbeffad3decdfacd9a51a4a97c07b7ab8969af5aa8bd51e43", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.004002094268798828, |
| "action_logits_max_abs_error": 5.021240234375, |
| "probability_max_abs_error": 0.0006047629910134744, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "es", |
| "questions": 3, |
| "sequence_length": 75, |
| "input_sha256": "3d429a6d7f31bc59256e88c56a1d7a8ae40a05c6277fc6d22341bdc2203d93b6", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.006280422210693359, |
| "action_logits_max_abs_error": 2.7939453125, |
| "probability_max_abs_error": 0.0015573923004306156, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "hi", |
| "questions": 3, |
| "sequence_length": 119, |
| "input_sha256": "23f2a78bed3a972ae02a7d3dc26e5b0eec62aee4230ff0c5a1388d7f044dd121", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.04454207420349121, |
| "action_logits_max_abs_error": 10.7470703125, |
| "probability_max_abs_error": 0.005444271459395356, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "ja", |
| "questions": 3, |
| "sequence_length": 75, |
| "input_sha256": "e73ae1efad1e50525b04f6a8f8004ccc15478f73ac3618fbe752e25c1c6c150b", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.0074710845947265625, |
| "action_logits_max_abs_error": 1.3525390625, |
| "probability_max_abs_error": 0.0009152261989081867, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "ru", |
| "questions": 3, |
| "sequence_length": 85, |
| "input_sha256": "bac6676f433fdbb4aa2b71dda957299b7e5a9efa2c2b526f000f51d8d4be3a08", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.005573272705078125, |
| "action_logits_max_abs_error": 2.29296875, |
| "probability_max_abs_error": 0.000573146121867707, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "empty_state", |
| "questions": 3, |
| "sequence_length": 46, |
| "input_sha256": "85f8a747931b65f225679d74c32afc65b23c679549560503921a969453d2c4e1", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.0027773380279541016, |
| "action_logits_max_abs_error": 2.47705078125, |
| "probability_max_abs_error": 0.00039989655194616613, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "long", |
| "questions": 3, |
| "sequence_length": 512, |
| "input_sha256": "029a742cb223bbb34add55cb46a0d266345a2e448f314ced18033f3710a9c8c7", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.0040493011474609375, |
| "action_logits_max_abs_error": 2.482421875, |
| "probability_max_abs_error": 0.00038620949119971826, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "conversation", |
| "questions": 3, |
| "sequence_length": 72, |
| "input_sha256": "a429d0393a0bd2af5d960fa45ab286d6741b89969693fa852560a6c87309a4d0", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.004204750061035156, |
| "action_logits_max_abs_error": 2.8369140625, |
| "probability_max_abs_error": 0.001078521392323395, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "mask_literals", |
| "questions": 3, |
| "sequence_length": 55, |
| "input_sha256": "f89667d91cd59690e55b05e6af9368ea1e27ef42829525a97d4cd3cace2f1989", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.020479202270507812, |
| "action_logits_max_abs_error": 2.844482421875, |
| "probability_max_abs_error": 0.0006111239970967852, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "many_questions", |
| "questions": 20, |
| "sequence_length": 93, |
| "input_sha256": "47942741e9ed5fabc4c49464b633b43608c61d04c9bc3f23fea5c0ae4da23ae3", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.0018677711486816406, |
| "action_logits_max_abs_error": 5.8330078125, |
| "probability_max_abs_error": 0.00016821344275153027, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 20, |
| "public_result_equal": false |
| }, |
| { |
| "case": "structured", |
| "questions": 3, |
| "sequence_length": 78, |
| "input_sha256": "8ba1aac3447918529ea99ae8c34146ebe16db359e7787412e6dda90c34f9b8b2", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.0077322423458099365, |
| "action_logits_max_abs_error": 1.15966796875, |
| "probability_max_abs_error": 0.0005516764917031236, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 3, |
| "public_result_equal": false |
| }, |
| { |
| "case": "twenty_options", |
| "questions": 1, |
| "sequence_length": 137, |
| "input_sha256": "f20fe7490e99ad43d4d1c5a9657f095968b904c9a60e8b3e02fce0544126f838", |
| "tokens_equal": true, |
| "logits_max_abs_error": 0.005983829498291016, |
| "action_logits_max_abs_error": 1.384765625, |
| "probability_max_abs_error": 1.541733241969172e-31, |
| "action_probability_max_abs_error": 0.0, |
| "argmax_agreements": 1, |
| "public_result_equal": true |
| } |
| ], |
| "stability": { |
| "calls": 100, |
| "elapsed_seconds": 3.521159500000067, |
| "finite": true, |
| "deterministic": true, |
| "active_bytes_before": 842602138, |
| "active_bytes_after": 842602138, |
| "active_growth_bytes": 0, |
| "peak_bytes": 1648506739 |
| } |
| } |
| ] |
| } |
|
|
