| { |
| "scope": "Local web runtime integration checks, not a general accuracy or speed benchmark. Full-video expected values are prior baseline outputs, not independently annotated ground truth.", |
| "cases": [ |
| { |
| "name": "skunks_full_soundtrack", |
| "spec": { |
| "text": "", |
| "questions": { |
| "Baby skunks": { |
| "type": "score", |
| "instructions": "Count the baby skunks visible at the same time in this video. Do not count the same animal again across frames. Choose the closest count.", |
| "criteria": [ |
| "0 baby skunks", |
| "1 baby skunks", |
| "2 baby skunks", |
| "3 baby skunks", |
| "4 baby skunks", |
| "5 baby skunks", |
| "6 baby skunks", |
| "7 baby skunks", |
| "8 baby skunks", |
| "9 baby skunks" |
| ] |
| }, |
| "Dog": { |
| "type": "score", |
| "instructions": "Count the dogs visible at the same time in this video. Do not count the same animal again across frames. Choose the closest count.", |
| "criteria": [ |
| "0 dogs", |
| "1 dogs", |
| "2 dogs", |
| "3 dogs", |
| "4 dogs", |
| "5 dogs", |
| "6 dogs", |
| "7 dogs", |
| "8 dogs", |
| "9 dogs" |
| ] |
| } |
| }, |
| "media": [ |
| { |
| "kind": "video", |
| "name": "skunks-full.mp4" |
| } |
| ] |
| }, |
| "media_sha256": "94860ab16790f8a4e6ae9c86c97e35322e1e4f58e2695c593dca0d5c5b170148", |
| "expected": { |
| "Baby skunks": 5, |
| "Dog": 1 |
| }, |
| "result": { |
| "answers": { |
| "Baby skunks": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_json_value_likelihoods", |
| "diagnostics": { |
| "logits": { |
| "0": 15.440519332885742, |
| "1": 18.91536521911621, |
| "2": 17.52647590637207, |
| "3": 17.263118743896484, |
| "4": 17.27484130859375, |
| "5": 25.10904312133789, |
| "6": 19.844194412231445, |
| "7": 18.685335159301758, |
| "8": 18.2138671875, |
| "9": 19.023639678955078 |
| }, |
| "allowed_token_mass": 0.9999809265136719, |
| "input_tokens": 2442 |
| }, |
| "confidence": 0.9591435032147506, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "probabilities": { |
| "0": 6.240175840612261e-05, |
| "1": 0.002015130710062158, |
| "2": 0.0005024770805513657, |
| "3": 0.000386137245892033, |
| "4": 0.00039069039991983777, |
| "5": 0.9866959764138694, |
| "6": 0.005101391037174207, |
| "7": 0.0016010409348640024, |
| "8": 0.0009991863613793212, |
| "9": 0.002245568057881544 |
| }, |
| "type": "score", |
| "score": 5.00924037645693, |
| "legend": { |
| "0": "0 baby skunks", |
| "1": "1 baby skunks", |
| "2": "2 baby skunks", |
| "3": "3 baby skunks", |
| "4": "4 baby skunks", |
| "5": "5 baby skunks", |
| "6": "6 baby skunks", |
| "7": "7 baby skunks", |
| "8": "8 baby skunks", |
| "9": "9 baby skunks" |
| } |
| }, |
| "Dog": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_json_value_likelihoods", |
| "diagnostics": { |
| "logits": { |
| "0": 16.827749252319336, |
| "1": 24.871902465820312, |
| "2": 18.862998962402344, |
| "3": 17.827789306640625, |
| "4": 18.372800827026367, |
| "5": 17.98102569580078, |
| "6": 16.684083938598633, |
| "7": 16.508058547973633, |
| "8": 17.43865203857422, |
| "9": 19.051477432250977 |
| }, |
| "allowed_token_mass": 0.9999866485595703, |
| "input_tokens": 2440 |
| }, |
| "confidence": 0.9671310741513678, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "probabilities": { |
| "0": 0.0003177193855978654, |
| "1": 0.9898629433627016, |
| "2": 0.0024318760318991664, |
| "3": 0.000863685426060581, |
| "4": 0.0014895362854541818, |
| "5": 0.0010067121486512969, |
| "6": 0.00027520141403579993, |
| "7": 0.0002307829950513383, |
| "8": 0.0005852688675128792, |
| "9": 0.0029362740830353746 |
| }, |
| "type": "score", |
| "score": 1.0426847647267505, |
| "legend": { |
| "0": "0 dogs", |
| "1": "1 dogs", |
| "2": "2 dogs", |
| "3": "3 dogs", |
| "4": "4 dogs", |
| "5": "5 dogs", |
| "6": "6 dogs", |
| "7": "7 dogs", |
| "8": "8 dogs", |
| "9": "9 dogs" |
| } |
| } |
| }, |
| "inference_seconds": 3.0780157500121277, |
| "execution": { |
| "execution": "shared_json_prefix_gpu_batched_fields", |
| "prefix_prefills": 1, |
| "prefix_tokens": 2436, |
| "primitive_fields": 2, |
| "question_suffix_tokens": [ |
| 6, |
| 4 |
| ], |
| "candidate_token_lengths": [ |
| [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ], |
| [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ] |
| ], |
| "preprocess_seconds": 0.6728912920225412, |
| "prefill_seconds": 2.3704014160030056, |
| "branch_seconds": 0.0345094169897493, |
| "branch_batch_sizes": [ |
| 2 |
| ], |
| "candidate_batch_sizes": [], |
| "compute_dtype": "float32", |
| "kv_storage": "replicated_per_batch_row", |
| "conditioning": "shared complete question schema; no previous field answers" |
| }, |
| "valid": true, |
| "total_seconds": 6.285174749995349, |
| "media": [ |
| { |
| "name": "skunks-full.mp4", |
| "kind": "video", |
| "bytes": 2935259, |
| "duration_seconds": 21.161042, |
| "has_soundtrack": true |
| } |
| ], |
| "media_prepare_seconds": 0.05148224998265505, |
| "decision_seconds": 3.0780157500121277, |
| "model": "Gemma 4 E2B \u00b7 frozen scorer", |
| "load_seconds": 3.548284250020515, |
| "comparison": { |
| "seconds": { |
| "parallel": 3.1294979999947827, |
| "normal": 3.179176165984245 |
| }, |
| "normal_over_parallel": 1.0158741644792695, |
| "parallel_values": { |
| "Baby skunks": 5, |
| "Dog": 1 |
| }, |
| "normal": { |
| "answers": { |
| "Baby skunks": 5, |
| "Dog": 1 |
| }, |
| "raw_text": "{\"Baby skunks\": 5, \"Dog\": 1}", |
| "valid": true, |
| "error": null, |
| "finish_reason": "stop", |
| "input_tokens": 2436, |
| "output_tokens": 15, |
| "max_output_tokens": 128, |
| "preprocess_seconds": 0.5861692079924978, |
| "generation_seconds": 2.5414814579999074, |
| "inference_seconds": 3.12769391600159, |
| "total_seconds": 3.179176165984245 |
| }, |
| "agreement": { |
| "Baby skunks": true, |
| "Dog": true |
| }, |
| "methodology": { |
| "model": "Same resident Gemma 4 E2B 4-bit weights; float32 compute; all 35 layers", |
| "timing": "One run per path, parallel scorer first, normal generation second. Includes input preparation and inference; shared upload decoding added equally to each path. Excludes model loading, upload transfer, and allocator reset. No warm-up runs; first-use effects and run order can affect this observation.", |
| "cache": "Fresh input KV and media features for every run; ordinary output-token KV caching remains enabled for normal generation.", |
| "answers": "Normal Gemma generates one JSON object for all fields, greedily, without thinking. Compare choice names, most likely grade levels, and booleans at a 50% threshold. The scorer also returns probability distributions and expected grades. Agreement is not an accuracy measurement." |
| } |
| }, |
| "request_seconds": 6.292751375003718 |
| } |
| }, |
| { |
| "name": "speech_skunks", |
| "spec": { |
| "text": "", |
| "media": [ |
| { |
| "kind": "audio", |
| "name": "skunks.wav" |
| } |
| ], |
| "questions": { |
| "skunks": { |
| "type": "score", |
| "instructions": "How many baby skunks does the speaker say there are?", |
| "criteria": [ |
| "0 baby skunks", |
| "1 baby skunks", |
| "2 baby skunks", |
| "3 baby skunks", |
| "4 baby skunks", |
| "5 baby skunks", |
| "6 baby skunks", |
| "7 baby skunks", |
| "8 baby skunks", |
| "9 baby skunks" |
| ] |
| }, |
| "dogs": { |
| "type": "score", |
| "instructions": "How many dogs does the speaker say there are?", |
| "criteria": [ |
| "0 dogs", |
| "1 dogs", |
| "2 dogs", |
| "3 dogs", |
| "4 dogs", |
| "5 dogs", |
| "6 dogs", |
| "7 dogs", |
| "8 dogs", |
| "9 dogs" |
| ] |
| }, |
| "call_vet": { |
| "type": "noul", |
| "instructions": "Does the speaker request that we call the vet?" |
| }, |
| "action": { |
| "type": "choice", |
| "instructions": "What does the speaker ask us to do?", |
| "criteria": { |
| "Call the vet": "Call the vet", |
| "Do not call the vet": "Do not call the vet" |
| } |
| } |
| } |
| }, |
| "media_sha256": "2b4e20e573eab19383d212d0bc718f97c0ad2da1c9c9120c506a5f17753abddc", |
| "expected": { |
| "skunks": 5, |
| "dogs": 1, |
| "call_vet": false, |
| "action": "Do not call the vet" |
| }, |
| "result": { |
| "answers": { |
| "skunks": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_json_value_likelihoods", |
| "diagnostics": { |
| "logits": { |
| "0": 2.4332430362701416, |
| "1": 0.33744150400161743, |
| "2": 0.49974024295806885, |
| "3": 1.0511709451675415, |
| "4": 2.741050958633423, |
| "5": 20.806188583374023, |
| "6": 2.202913999557495, |
| "7": -1.8478120565414429, |
| "8": -2.2469234466552734, |
| "9": 0.48146724700927734 |
| }, |
| "allowed_token_mass": 1.0, |
| "input_tokens": 620 |
| }, |
| "confidence": 0.9999996565698799, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "probabilities": { |
| "0": 1.0488928261791463e-08, |
| "1": 1.2898406934682289e-09, |
| "2": 1.5171255458984183e-09, |
| "3": 2.6333272442131735e-09, |
| "4": 1.4269553933267914e-08, |
| "5": 0.9999999597381529, |
| "6": 8.331064281204806e-09, |
| "7": 1.4504157649251244e-10, |
| "8": 9.731070908143261e-11, |
| "9": 1.4896548671182722e-09 |
| }, |
| "type": "score", |
| "score": 4.999999933180111, |
| "legend": { |
| "0": "0 baby skunks", |
| "1": "1 baby skunks", |
| "2": "2 baby skunks", |
| "3": "3 baby skunks", |
| "4": "4 baby skunks", |
| "5": "5 baby skunks", |
| "6": "6 baby skunks", |
| "7": "7 baby skunks", |
| "8": "8 baby skunks", |
| "9": "9 baby skunks" |
| } |
| }, |
| "dogs": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_json_value_likelihoods", |
| "diagnostics": { |
| "logits": { |
| "0": 12.110952377319336, |
| "1": 25.3037166595459, |
| "2": 9.88134765625, |
| "3": 6.69423246383667, |
| "4": 8.310088157653809, |
| "5": 7.257328510284424, |
| "6": 6.1623053550720215, |
| "7": 4.915668487548828, |
| "8": 5.334745407104492, |
| "9": 7.838923931121826 |
| }, |
| "allowed_token_mass": 1.0, |
| "input_tokens": 619 |
| }, |
| "confidence": 0.9999862804353773, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "probabilities": { |
| "0": 1.86403615456672e-06, |
| "1": 0.9999978365667048, |
| "2": 2.005161255538126e-07, |
| "3": 8.279474314556474e-09, |
| "4": 4.16639052158653e-08, |
| "5": 1.4539593643851128e-08, |
| "6": 4.863957315537724e-09, |
| "7": 1.3982416759546908e-09, |
| "8": 2.126106602756648e-09, |
| "9": 2.6009736406271668e-08 |
| }, |
| "type": "score", |
| "score": 0.9999987918588841, |
| "legend": { |
| "0": "0 dogs", |
| "1": "1 dogs", |
| "2": "2 dogs", |
| "3": "3 dogs", |
| "4": "4 dogs", |
| "5": "5 dogs", |
| "6": "6 dogs", |
| "7": "7 dogs", |
| "8": "8 dogs", |
| "9": "9 dogs" |
| } |
| }, |
| "call_vet": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_json_value_likelihoods", |
| "diagnostics": { |
| "logits": { |
| "true": -0.8286030292510986, |
| "false": 6.238976955413818 |
| }, |
| "allowed_token_mass": 0.999083399772644, |
| "input_tokens": 620 |
| }, |
| "type": "noul", |
| "noul": 0.0008515673929433828 |
| }, |
| "action": { |
| "calibration_status": "unvalidated", |
| "temperature": 1.0, |
| "probability_source": "restricted_json_value_likelihoods", |
| "diagnostics": { |
| "logits": { |
| "Call the vet": -40.304771423339844, |
| "Do not call the vet": -23.257143020629883 |
| }, |
| "allowed_token_mass": 7.93507687306582e-11, |
| "input_tokens": 624 |
| }, |
| "confidence": 0.9999989722115856, |
| "confidence_definition": "one_minus_normalized_entropy", |
| "probabilities": { |
| "Call the vet": 3.947380923962134e-08, |
| "Do not call the vet": 0.9999999605261907 |
| }, |
| "type": "choice", |
| "choice": "Do not call the vet", |
| "selected_probability": 0.9999999605261907 |
| } |
| }, |
| "inference_seconds": 0.27987883301102556, |
| "execution": { |
| "execution": "shared_json_prefix_gpu_batched_fields", |
| "prefix_prefills": 1, |
| "prefix_tokens": 615, |
| "primitive_fields": 4, |
| "question_suffix_tokens": [ |
| 5, |
| 4, |
| 5, |
| 4 |
| ], |
| "candidate_token_lengths": [ |
| [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ], |
| [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ], |
| [ |
| 1, |
| 1 |
| ], |
| [ |
| 4, |
| 6 |
| ] |
| ], |
| "preprocess_seconds": 0.043527457979507744, |
| "prefill_seconds": 0.153558875026647, |
| "branch_seconds": 0.08244479197310284, |
| "branch_batch_sizes": [ |
| 3 |
| ], |
| "candidate_batch_sizes": [ |
| 2 |
| ], |
| "compute_dtype": "float32", |
| "kv_storage": "replicated_per_batch_row", |
| "conditioning": "shared complete question schema; no previous field answers" |
| }, |
| "valid": true, |
| "total_seconds": 1.0800352079968434, |
| "media": [ |
| { |
| "name": "skunks.wav", |
| "kind": "audio", |
| "bytes": 148568, |
| "duration_seconds": 4.51475, |
| "has_soundtrack": false |
| } |
| ], |
| "media_prepare_seconds": 0.10681216599186882, |
| "decision_seconds": 0.27987883301102556, |
| "model": "Gemma 4 E2B \u00b7 frozen scorer", |
| "load_seconds": 3.548284250020515, |
| "comparison": { |
| "seconds": { |
| "parallel": 0.3866909990028944, |
| "normal": 0.7924858739716001 |
| }, |
| "normal_over_parallel": 2.0494034668897694, |
| "parallel_values": { |
| "skunks": 5, |
| "dogs": 1, |
| "call_vet": false, |
| "action": "Do not call the vet" |
| }, |
| "normal": { |
| "answers": { |
| "skunks": 5, |
| "dogs": 1, |
| "call_vet": false, |
| "action": "Do not call the vet" |
| }, |
| "raw_text": "{\"skunks\": 5, \"dogs\": 1, \"call_vet\": false, \"action\": \"Do not call the vet\"}", |
| "valid": true, |
| "error": null, |
| "finish_reason": "stop", |
| "input_tokens": 615, |
| "output_tokens": 31, |
| "max_output_tokens": 128, |
| "preprocess_seconds": 0.03355474999989383, |
| "generation_seconds": 0.6520849579828791, |
| "inference_seconds": 0.6856737079797313, |
| "total_seconds": 0.7924858739716001 |
| }, |
| "agreement": { |
| "skunks": true, |
| "dogs": true, |
| "call_vet": true, |
| "action": true |
| }, |
| "methodology": { |
| "model": "Same resident Gemma 4 E2B 4-bit weights; float32 compute; all 35 layers", |
| "timing": "One run per path, parallel scorer first, normal generation second. Includes input preparation and inference; shared upload decoding added equally to each path. Excludes model loading, upload transfer, and allocator reset. No warm-up runs; first-use effects and run order can affect this observation.", |
| "cache": "Fresh input KV and media features for every run; ordinary output-token KV caching remains enabled for normal generation.", |
| "answers": "Normal Gemma generates one JSON object for all fields, greedily, without thinking. Compare choice names, most likely grade levels, and booleans at a 50% threshold. The scorer also returns probability distributions and expected grades. Agreement is not an accuracy measurement." |
| } |
| }, |
| "request_seconds": 1.0820253329875413 |
| } |
| } |
| ] |
| } |
|
|
