mlboydaisuke's picture
Granite-Embedding-97M-Multilingual-R2: fp32 + w8 bundles, S=128/512, macOS JIT + iPhone 17 Pro h18p AOT, tokenizer, reference fixtures, provenance
0807d1b verified
Raw
History Blame Contribute Delete
20.5 kB
{
"status": "PASS",
"thresholds": {
"cosine_min": 0.999,
"max_abs": 0.02,
"norm_error_max": 0.002,
"similarity_error_max": 0.01,
"top1": "exact for every query",
"pairwise_order": "exact when oracle gap >=0.001"
},
"failures": [],
"per_fixture": {
"empty": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998006,
"max_abs": 2.980232238769531e-07,
"rmse": 3.287376472455382e-08,
"reference_norm": 1.0000001060237258,
"candidate_norm": 0.9999999778707191,
"norm_error": 1.281530066510328e-07
},
"short": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998512,
"max_abs": 1.043081283569336e-07,
"rmse": 2.8067909054751824e-08,
"reference_norm": 0.9999999780118689,
"candidate_norm": 1.0000000476578494,
"norm_error": 6.964598053826876e-08
},
"whitespace": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999996697,
"max_abs": 1.7136335372924805e-07,
"rmse": 4.158988696649394e-08,
"reference_norm": 0.9999999691327314,
"candidate_norm": 1.0000000306026646,
"norm_error": 6.146993325462091e-08
},
"whitespace_edges": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998388,
"max_abs": 1.0803341865539551e-07,
"rmse": 2.9065738742053113e-08,
"reference_norm": 1.000000048862692,
"candidate_norm": 1.0000000062866545,
"norm_error": 4.2576037362351826e-08
},
"unicode": {
"finite": true,
"shape_match": true,
"cosine": 0.999999999999835,
"max_abs": 1.4156103134155273e-07,
"rmse": 2.938501641068203e-08,
"reference_norm": 1.000000003099415,
"candidate_norm": 1.000000045849268,
"norm_error": 4.274985299090872e-08
},
"english": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998407,
"max_abs": 1.043081283569336e-07,
"rmse": 2.8913140276246598e-08,
"reference_norm": 0.9999999978182623,
"candidate_norm": 0.9999999525601833,
"norm_error": 4.52580790621937e-08
},
"japanese": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997677,
"max_abs": 1.4156103134155273e-07,
"rmse": 3.489241665301136e-08,
"reference_norm": 1.000000032888026,
"candidate_norm": 0.9999999786437835,
"norm_error": 5.4244242542189625e-08
},
"mixed": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998117,
"max_abs": 1.1920928955078125e-07,
"rmse": 3.1309528541713154e-08,
"reference_norm": 1.0000000084772873,
"candidate_norm": 1.0000000080999096,
"norm_error": 3.7737768465717636e-10
},
"q_train": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998315,
"max_abs": 7.799826562404633e-08,
"rmse": 2.9638772321580017e-08,
"reference_norm": 1.0000000041141424,
"candidate_norm": 1.000000015703183,
"norm_error": 1.1589040616044599e-08
},
"q_refund": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998883,
"max_abs": 7.450580596923828e-08,
"rmse": 2.4176543060275505e-08,
"reference_norm": 1.0000000166590979,
"candidate_norm": 0.9999999934829632,
"norm_error": 2.3176134678060123e-08
},
"q_backup": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998206,
"max_abs": 8.940696716308594e-08,
"rmse": 3.056609367590288e-08,
"reference_norm": 1.0000000218807725,
"candidate_norm": 1.000000025504604,
"norm_error": 3.6238314571335195e-09
},
"q_garden": {
"finite": true,
"shape_match": true,
"cosine": 0.999999999999826,
"max_abs": 1.30385160446167e-07,
"rmse": 3.011158801546051e-08,
"reference_norm": 0.999999965733353,
"candidate_norm": 0.9999999874526099,
"norm_error": 2.1719256837471335e-08
},
"d_train": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997067,
"max_abs": 1.2712553143501282e-07,
"rmse": 3.9342484587263154e-08,
"reference_norm": 1.0000000726503868,
"candidate_norm": 0.9999999869972638,
"norm_error": 8.565312303865369e-08
},
"d_train_negative1": {
"finite": true,
"shape_match": true,
"cosine": 0.999999999999773,
"max_abs": 1.4156103134155273e-07,
"rmse": 3.438570826684839e-08,
"reference_norm": 0.9999999722433937,
"candidate_norm": 0.9999999827409982,
"norm_error": 1.0497604563397545e-08
},
"d_train_negative2": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998113,
"max_abs": 1.7881393432617188e-07,
"rmse": 3.19191147954468e-08,
"reference_norm": 0.9999999156522698,
"candidate_norm": 1.0000000326126801,
"norm_error": 1.1696041035680338e-07
},
"d_refund": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998572,
"max_abs": 9.499490261077881e-08,
"rmse": 2.73821694675214e-08,
"reference_norm": 1.000000033703324,
"candidate_norm": 0.9999999892749887,
"norm_error": 4.442833523299328e-08
},
"d_refund_negative1": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998519,
"max_abs": 1.043081283569336e-07,
"rmse": 2.8031353727524345e-08,
"reference_norm": 1.0000000240324858,
"candidate_norm": 0.9999999507490022,
"norm_error": 7.328348361479442e-08
},
"d_refund_negative2": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998923,
"max_abs": 7.450580596923828e-08,
"rmse": 2.369956656760069e-08,
"reference_norm": 0.9999999895651434,
"candidate_norm": 1.0000000132001876,
"norm_error": 2.3635044255065907e-08
},
"d_backup": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997382,
"max_abs": 1.2293457984924316e-07,
"rmse": 3.695138941529502e-08,
"reference_norm": 1.0000000218414224,
"candidate_norm": 1.0000000336540507,
"norm_error": 1.1812628208929254e-08
},
"d_backup_negative1": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997886,
"max_abs": 1.341104507446289e-07,
"rmse": 3.33508158760871e-08,
"reference_norm": 1.0000000822872175,
"candidate_norm": 1.0000000145885868,
"norm_error": 6.769863070132942e-08
},
"d_backup_negative2": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997676,
"max_abs": 1.7881393432617188e-07,
"rmse": 3.5024147734315726e-08,
"reference_norm": 0.9999999299917344,
"candidate_norm": 1.0000000119544727,
"norm_error": 8.19627383741306e-08
},
"d_garden": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997793,
"max_abs": 1.1920928955078125e-07,
"rmse": 3.4010256895752575e-08,
"reference_norm": 1.0000000155656608,
"candidate_norm": 1.000000065913024,
"norm_error": 5.03473631674467e-08
},
"d_garden_negative1": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998488,
"max_abs": 1.0139774531126022e-07,
"rmse": 2.816380702909151e-08,
"reference_norm": 1.0000000474984756,
"candidate_norm": 0.9999999990250277,
"norm_error": 4.847344792224817e-08
},
"d_garden_negative2": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999996807,
"max_abs": 1.5273690223693848e-07,
"rmse": 4.105429801530387e-08,
"reference_norm": 0.9999998974674439,
"candidate_norm": 0.9999999914962371,
"norm_error": 9.402879319608815e-08
},
"boundary_62": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998749,
"max_abs": 8.940696716308594e-08,
"rmse": 2.5559136699064088e-08,
"reference_norm": 0.999999998282109,
"candidate_norm": 0.9999999875190154,
"norm_error": 1.0763093527543788e-08
},
"boundary_63": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998341,
"max_abs": 2.1047890186309814e-07,
"rmse": 2.967026047181605e-08,
"reference_norm": 0.9999999667560612,
"candidate_norm": 1.0000000431587608,
"norm_error": 7.640269961139978e-08
},
"boundary_64": {
"finite": true,
"shape_match": true,
"cosine": 0.999999999999674,
"max_abs": 1.2293457984924316e-07,
"rmse": 4.1198044311847425e-08,
"reference_norm": 0.9999999823000086,
"candidate_norm": 0.9999999943451501,
"norm_error": 1.2045141439820384e-08
},
"boundary_126": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997158,
"max_abs": 1.7881393432617188e-07,
"rmse": 3.901084419855879e-08,
"reference_norm": 1.0000001078695948,
"candidate_norm": 0.9999999822187703,
"norm_error": 1.2565082452287157e-07
},
"boundary_127": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997158,
"max_abs": 1.7881393432617188e-07,
"rmse": 3.901084419855879e-08,
"reference_norm": 1.0000001078695948,
"candidate_norm": 0.9999999822187703,
"norm_error": 1.2565082452287157e-07
},
"boundary_128": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997158,
"max_abs": 1.7881393432617188e-07,
"rmse": 3.901084419855879e-08,
"reference_norm": 1.0000001078695948,
"candidate_norm": 0.9999999822187703,
"norm_error": 1.2565082452287157e-07
},
"boundary_510": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997158,
"max_abs": 1.7881393432617188e-07,
"rmse": 3.901084419855879e-08,
"reference_norm": 1.0000001078695948,
"candidate_norm": 0.9999999822187703,
"norm_error": 1.2565082452287157e-07
},
"boundary_511": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997158,
"max_abs": 1.7881393432617188e-07,
"rmse": 3.901084419855879e-08,
"reference_norm": 1.0000001078695948,
"candidate_norm": 0.9999999822187703,
"norm_error": 1.2565082452287157e-07
},
"boundary_512": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997158,
"max_abs": 1.7881393432617188e-07,
"rmse": 3.901084419855879e-08,
"reference_norm": 1.0000001078695948,
"candidate_norm": 0.9999999822187703,
"norm_error": 1.2565082452287157e-07
},
"boundary_700": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997158,
"max_abs": 1.7881393432617188e-07,
"rmse": 3.901084419855879e-08,
"reference_norm": 1.0000001078695948,
"candidate_norm": 0.9999999822187703,
"norm_error": 1.2565082452287157e-07
},
"long_japanese": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998515,
"max_abs": 9.12696123123169e-08,
"rmse": 2.7894878870698198e-08,
"reference_norm": 1.0000000277223837,
"candidate_norm": 0.9999999875427343,
"norm_error": 4.0179649385407856e-08
}
},
"retrieval": [
{
"query": "q_train",
"reference_order": [
"d_train",
"d_train_negative1",
"d_train_negative2",
"d_refund_negative1",
"d_refund",
"d_backup_negative1",
"d_refund_negative2",
"d_garden_negative2",
"d_garden_negative1",
"d_backup",
"d_garden",
"d_backup_negative2"
],
"candidate_order": [
"d_train",
"d_train_negative1",
"d_train_negative2",
"d_refund_negative1",
"d_refund",
"d_backup_negative1",
"d_refund_negative2",
"d_garden_negative2",
"d_garden_negative1",
"d_backup",
"d_garden",
"d_backup_negative2"
],
"reference_scores": [
0.9345145093980466,
0.9170717825497254,
0.911723885423255,
0.7317248664701558,
0.7513318377518619,
0.7041310675303637,
0.6939833258662486,
0.7105373505197093,
0.6719489263206282,
0.6797689367126396,
0.6965976234455196,
0.6990397887988085
],
"candidate_scores": [
0.9345143686279008,
0.9170717514458875,
0.9117240332177375,
0.7317249458777333,
0.7513318059082994,
0.7041311040249879,
0.693983343872367,
0.7105373775608574,
0.6719491203114627,
0.6797689947757443,
0.6965975298279861,
0.6990397691950453
],
"reference_margin": 0.017442726848321155,
"candidate_margin": 0.017442617182013342,
"max_score_error": 1.9399083450721832e-07,
"top1_match": true,
"clear_pair_flips": 0
},
{
"query": "q_refund",
"reference_order": [
"d_refund",
"d_refund_negative2",
"d_refund_negative1",
"d_backup",
"d_train",
"d_train_negative1",
"d_train_negative2",
"d_backup_negative2",
"d_backup_negative1",
"d_garden",
"d_garden_negative2",
"d_garden_negative1"
],
"candidate_order": [
"d_refund",
"d_refund_negative2",
"d_refund_negative1",
"d_backup",
"d_train",
"d_train_negative1",
"d_train_negative2",
"d_backup_negative2",
"d_backup_negative1",
"d_garden",
"d_garden_negative2",
"d_garden_negative1"
],
"reference_scores": [
0.7214556217422334,
0.7144578286633229,
0.7132234614371428,
0.9581554437009046,
0.8592080211421944,
0.92480017781159,
0.7215637240319225,
0.67376318167836,
0.6822757076115306,
0.6410706404685216,
0.6339530415617187,
0.6408546761798151
],
"candidate_scores": [
0.721455566622423,
0.7144578843444396,
0.7132236082004464,
0.95815538242277,
0.8592079442111474,
0.9248001718266805,
0.7215637684516474,
0.6737631978041144,
0.6822757724812999,
0.6410707351087964,
0.6339530184577601,
0.6408546926210069
],
"reference_margin": 0.03335526588931459,
"candidate_margin": 0.03335521059608948,
"max_score_error": 1.4676330362117795e-07,
"top1_match": true,
"clear_pair_flips": 0
},
{
"query": "q_backup",
"reference_order": [
"d_backup",
"d_backup_negative1",
"d_backup_negative2",
"d_refund",
"d_garden_negative2",
"d_garden",
"d_garden_negative1",
"d_train",
"d_train_negative2",
"d_train_negative1",
"d_refund_negative2",
"d_refund_negative1"
],
"candidate_order": [
"d_backup",
"d_backup_negative1",
"d_backup_negative2",
"d_refund",
"d_garden_negative2",
"d_garden",
"d_garden_negative1",
"d_train",
"d_train_negative2",
"d_train_negative1",
"d_refund_negative2",
"d_refund_negative1"
],
"reference_scores": [
0.7052823369965103,
0.6912889675495203,
0.6988308676272218,
0.7384987491405721,
0.6797166343077331,
0.6886572358164231,
0.9409091202419188,
0.8921029303277296,
0.8615508288157594,
0.7076615329689144,
0.7066142858726151,
0.7239074247176029
],
"candidate_scores": [
0.7052822642195096,
0.6912890907585553,
0.6988310032511069,
0.7384988205591055,
0.679716644575597,
0.6886572531007281,
0.9409091493929017,
0.8921028995936139,
0.8615509402954667,
0.707661645406316,
0.7066143281821606,
0.7239074766395418
],
"reference_margin": 0.04880618991418917,
"candidate_margin": 0.048806249799287826,
"max_score_error": 1.3562388512244894e-07,
"top1_match": true,
"clear_pair_flips": 0
},
{
"query": "q_garden",
"reference_order": [
"d_garden",
"d_garden_negative1",
"d_garden_negative2",
"d_backup_negative1",
"d_backup",
"d_train_negative2",
"d_train_negative1",
"d_train",
"d_backup_negative2",
"d_refund",
"d_refund_negative1",
"d_refund_negative2"
],
"candidate_order": [
"d_garden",
"d_garden_negative1",
"d_garden_negative2",
"d_backup_negative1",
"d_backup",
"d_train_negative2",
"d_train_negative1",
"d_train",
"d_backup_negative2",
"d_refund",
"d_refund_negative1",
"d_refund_negative2"
],
"reference_scores": [
0.660547675288807,
0.6813342545261133,
0.6865840756506723,
0.6282090252676062,
0.6272401859204846,
0.6238906236445231,
0.6881865592172072,
0.7191951945129387,
0.6567628481463305,
0.9618270896483002,
0.9089393242690795,
0.8270065201373913
],
"candidate_scores": [
0.6605476483638335,
0.6813343293628769,
0.6865842134967477,
0.6282090474321012,
0.6272401951210228,
0.6238906562401113,
0.6881866057508155,
0.7191952374149132,
0.6567630324093215,
0.9618271402125943,
0.908939230763653,
0.8270065617213083
],
"reference_margin": 0.052887765379220686,
"candidate_margin": 0.052887909448941284,
"max_score_error": 1.8426299108753597e-07,
"top1_match": true,
"clear_pair_flips": 0
}
],
"min_cosine": 0.9999999999996697,
"max_abs": 2.980232238769531e-07,
"wrong_pairing_control": {
"status": "FAIL",
"min_cosine": 0.6582764187682952
},
"runtime_record": {
"allowed_compute_units": [
"cpu",
"gpu",
"neuralEngine"
],
"artifact": "/Users/majimadaisuke/code/codex-conversions/2026-09-16/coreai-granite97m/exports/macos/granite97m_fp32_s128_bound.aimodel",
"artifact_verification": "PASS",
"backend": "gpu_preferred",
"backend_observation": "GPU preference; actual per-op placement not exposed by this runner",
"completed_fixture_count": 35,
"coreai_device_architecture": "h16c",
"current_fixture": "long_japanese",
"finished_at": "2026-09-19T06:17:55Z",
"fixture_sha256": "bafddd86b4687383507bfacf0436ab743fc340560d7407d2859d2519a0799bab",
"gate_status": "NOT RUN: compare outputs with independent HF numerical and retrieval gate",
"load_cache_state": "not cleared; first load in this process only",
"load_ms": 480.701917,
"manifest_sha256": "c5cfab8395a0f73374ef856fb5ca782cba5bcf7dc64b2d3805d13bcda7f37db0",
"model_sha": "835ad14087e140460703cf0fae09f97d469d65c2",
"os_version": "Version 27.0 (Build 26A428)",
"phase": "completed",
"precision": "fp32",
"preprocessing": "exact HF fixture input_ids and attention_mask; no tokenizer",
"requested_backend": "gpu",
"runner_binary_sha256": "a43ce2e2e4c6f855d785bcaef43a56114258de3514a80cfab56829867fb54d53",
"runner_schema_version": 1,
"runner_source_sha256": "30c0935d62f2f349a6a5e635d45b32d61abd518b53d6aec2d1a5876ce693718f",
"selected_fixture_count": 35,
"sequence_length": 128,
"started_at": "2026-09-19T06:17:52Z",
"status": "COMPUTED",
"total_fixture_count": 35,
"trial": 3
},
"input_hashes": {
"results/r01_mac_gpu_s128.json": "1055b74621f805f92dc74e3fa6c8349dbc5747b4acf6184dae3a4fd0fcb06cf4",
"fixtures/golden_s128.json": "bafddd86b4687383507bfacf0436ab743fc340560d7407d2859d2519a0799bab",
"scripts/gate_metrics.py": "d4d73a64cb4f15850b1453ebe91beb4cfbf90056e8faaf10f5afb44991511c26",
"scripts/gate_returned.py": "d5605dbe65082207cf06516df2416117714cd874992c9870a71c434893509790"
}
}