mlboydaisuke's picture
Granite-Embedding-97M-Multilingual-R2: fp32 + w8 bundles, S=128/512, macOS JIT + iPhone 17 Pro h18p AOT, tokenizer, reference fixtures, provenance
0807d1b verified
Raw
History Blame Contribute Delete
20.5 kB
{
"status": "PASS",
"thresholds": {
"cosine_min": 0.999,
"max_abs": 0.02,
"norm_error_max": 0.002,
"similarity_error_max": 0.01,
"top1": "exact for every query",
"pairwise_order": "exact when oracle gap >=0.001"
},
"failures": [],
"per_fixture": {
"empty": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998435,
"max_abs": 2.384185791015625e-07,
"rmse": 2.872874704773288e-08,
"reference_norm": 1.0000001060237258,
"candidate_norm": 1.0000000396769693,
"norm_error": 6.634675653316435e-08
},
"short": {
"finite": true,
"shape_match": true,
"cosine": 0.999999999999815,
"max_abs": 1.043081283569336e-07,
"rmse": 3.107241505286738e-08,
"reference_norm": 0.9999999780118689,
"candidate_norm": 1.0000000088661969,
"norm_error": 3.0854327981444385e-08
},
"whitespace": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999996441,
"max_abs": 2.2351741790771484e-07,
"rmse": 4.338599207604499e-08,
"reference_norm": 0.9999999691327314,
"candidate_norm": 1.0000000737730979,
"norm_error": 1.0464036648283326e-07
},
"whitespace_edges": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997963,
"max_abs": 1.0803341865539551e-07,
"rmse": 3.3086775219364376e-08,
"reference_norm": 1.000000048862692,
"candidate_norm": 0.9999999353312752,
"norm_error": 1.1353141671222033e-07
},
"unicode": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998364,
"max_abs": 9.313225746154785e-08,
"rmse": 2.9286154559237527e-08,
"reference_norm": 1.000000003099415,
"candidate_norm": 1.0000000532005007,
"norm_error": 5.010108572456318e-08
},
"english": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998848,
"max_abs": 8.568167686462402e-08,
"rmse": 2.4514131857601353e-08,
"reference_norm": 0.9999999978182623,
"candidate_norm": 1.0000000082433582,
"norm_error": 1.0425095897659276e-08
},
"japanese": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999996647,
"max_abs": 1.341104507446289e-07,
"rmse": 4.177564374373213e-08,
"reference_norm": 1.000000032888026,
"candidate_norm": 1.0000000138348684,
"norm_error": 1.9053157718218472e-08
},
"mixed": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998219,
"max_abs": 1.1920928955078125e-07,
"rmse": 3.044897636700764e-08,
"reference_norm": 1.0000000084772873,
"candidate_norm": 0.9999999917822434,
"norm_error": 1.6695043902892337e-08
},
"q_train": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997655,
"max_abs": 1.3783574104309082e-07,
"rmse": 3.4948398355365716e-08,
"reference_norm": 1.0000000041141424,
"candidate_norm": 1.0000000070248447,
"norm_error": 2.9107023458152526e-09
},
"q_refund": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998863,
"max_abs": 8.568167686462402e-08,
"rmse": 2.4378432404407474e-08,
"reference_norm": 1.0000000166590979,
"candidate_norm": 1.0000000474982083,
"norm_error": 3.083911037649045e-08
},
"q_backup": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998204,
"max_abs": 9.685754776000977e-08,
"rmse": 3.0588956998522086e-08,
"reference_norm": 1.0000000218807725,
"candidate_norm": 1.0000000264403377,
"norm_error": 4.559565169870439e-09
},
"q_garden": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998335,
"max_abs": 1.3783574104309082e-07,
"rmse": 2.9484625070784295e-08,
"reference_norm": 0.999999965733353,
"candidate_norm": 0.9999999865687327,
"norm_error": 2.0835379643813212e-08
},
"d_train": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997347,
"max_abs": 1.6391277313232422e-07,
"rmse": 3.747696994957748e-08,
"reference_norm": 1.0000000726503868,
"candidate_norm": 0.999999981136755,
"norm_error": 9.151363178361294e-08
},
"d_train_negative1": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999995737,
"max_abs": 1.6391277313232422e-07,
"rmse": 4.713964318479129e-08,
"reference_norm": 0.9999999722433937,
"candidate_norm": 1.0000000057072498,
"norm_error": 3.346385613056668e-08
},
"d_train_negative2": {
"finite": true,
"shape_match": true,
"cosine": 0.99999999999983,
"max_abs": 1.1920928955078125e-07,
"rmse": 3.0236353228744924e-08,
"reference_norm": 0.9999999156522698,
"candidate_norm": 1.000000020774052,
"norm_error": 1.0512178227894964e-07
},
"d_refund": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998634,
"max_abs": 9.685754776000977e-08,
"rmse": 2.6767394053456998e-08,
"reference_norm": 1.000000033703324,
"candidate_norm": 0.9999999958402466,
"norm_error": 3.786307734010563e-08
},
"d_refund_negative1": {
"finite": true,
"shape_match": true,
"cosine": 0.999999999999845,
"max_abs": 1.7881393432617188e-07,
"rmse": 2.8490839706893932e-08,
"reference_norm": 1.0000000240324858,
"candidate_norm": 0.9999999782103024,
"norm_error": 4.582218338100574e-08
},
"d_refund_negative2": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998597,
"max_abs": 1.1082738637924194e-07,
"rmse": 2.703732266034839e-08,
"reference_norm": 0.9999999895651434,
"candidate_norm": 0.9999999915598612,
"norm_error": 1.994717835351878e-09
},
"d_backup": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999996776,
"max_abs": 1.457519829273224e-07,
"rmse": 4.100298692675143e-08,
"reference_norm": 1.0000000218414224,
"candidate_norm": 1.000000038776749,
"norm_error": 1.6935326474509793e-08
},
"d_backup_negative1": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997471,
"max_abs": 1.0617077350616455e-07,
"rmse": 3.6380629510526005e-08,
"reference_norm": 1.0000000822872175,
"candidate_norm": 1.000000033555662,
"norm_error": 4.873155545759289e-08
},
"d_backup_negative2": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997896,
"max_abs": 1.1920928955078125e-07,
"rmse": 3.317241210413883e-08,
"reference_norm": 0.9999999299917344,
"candidate_norm": 0.9999999736170806,
"norm_error": 4.3625346224374084e-08
},
"d_garden": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998265,
"max_abs": 8.940696716308594e-08,
"rmse": 3.03946595069891e-08,
"reference_norm": 1.0000000155656608,
"candidate_norm": 0.9999999299594046,
"norm_error": 8.560625619491447e-08
},
"d_garden_negative1": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998428,
"max_abs": 1.1920928955078125e-07,
"rmse": 2.862599889532838e-08,
"reference_norm": 1.0000000474984756,
"candidate_norm": 1.00000004132397,
"norm_error": 6.174505573497413e-09
},
"d_garden_negative2": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997778,
"max_abs": 1.564621925354004e-07,
"rmse": 3.535400363187914e-08,
"reference_norm": 0.9999998974674439,
"candidate_norm": 1.0000000866711518,
"norm_error": 1.892037079231912e-07
},
"boundary_62": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998458,
"max_abs": 1.3224780559539795e-07,
"rmse": 2.8364122032690103e-08,
"reference_norm": 0.999999998282109,
"candidate_norm": 0.9999999750597186,
"norm_error": 2.322239034402429e-08
},
"boundary_63": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998347,
"max_abs": 1.564621925354004e-07,
"rmse": 2.9440102239545497e-08,
"reference_norm": 0.9999999667560612,
"candidate_norm": 1.0000000175809758,
"norm_error": 5.082491461028127e-08
},
"boundary_64": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999988683,
"max_abs": 2.5331974029541016e-07,
"rmse": 7.678363250591561e-08,
"reference_norm": 0.9999999823000086,
"candidate_norm": 1.0000000007876686,
"norm_error": 1.8487659958488223e-08
},
"boundary_126": {
"finite": true,
"shape_match": true,
"cosine": 0.99999999999974,
"max_abs": 2.980232238769531e-07,
"rmse": 3.7298847219921546e-08,
"reference_norm": 1.0000001078695948,
"candidate_norm": 0.9999999895938492,
"norm_error": 1.182757456419381e-07
},
"boundary_127": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999997753,
"max_abs": 1.1920928955078125e-07,
"rmse": 3.421072086615677e-08,
"reference_norm": 0.9999999947099483,
"candidate_norm": 1.0000000116262893,
"norm_error": 1.6916340994654888e-08
},
"boundary_128": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998264,
"max_abs": 1.1920928955078125e-07,
"rmse": 3.009174997796441e-08,
"reference_norm": 1.0000000064041927,
"candidate_norm": 1.000000033244877,
"norm_error": 2.6840684341777887e-08
},
"boundary_510": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998436,
"max_abs": 1.341104507446289e-07,
"rmse": 2.8564044247177016e-08,
"reference_norm": 0.9999999534388136,
"candidate_norm": 0.9999999787153682,
"norm_error": 2.527655451611821e-08
},
"boundary_511": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998436,
"max_abs": 1.341104507446289e-07,
"rmse": 2.8564044247177016e-08,
"reference_norm": 0.9999999534388136,
"candidate_norm": 0.9999999787153682,
"norm_error": 2.527655451611821e-08
},
"boundary_512": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998436,
"max_abs": 1.341104507446289e-07,
"rmse": 2.8564044247177016e-08,
"reference_norm": 0.9999999534388136,
"candidate_norm": 0.9999999787153682,
"norm_error": 2.527655451611821e-08
},
"boundary_700": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999998436,
"max_abs": 1.341104507446289e-07,
"rmse": 2.8564044247177016e-08,
"reference_norm": 0.9999999534388136,
"candidate_norm": 0.9999999787153682,
"norm_error": 2.527655451611821e-08
},
"long_japanese": {
"finite": true,
"shape_match": true,
"cosine": 0.9999999999991717,
"max_abs": 1.8998980522155762e-07,
"rmse": 6.577183640755026e-08,
"reference_norm": 0.9999999344743007,
"candidate_norm": 1.0000000011684225,
"norm_error": 6.669412178084144e-08
}
},
"retrieval": [
{
"query": "q_train",
"reference_order": [
"d_train",
"d_train_negative1",
"d_train_negative2",
"d_refund_negative1",
"d_refund",
"d_backup_negative1",
"d_refund_negative2",
"d_garden_negative2",
"d_garden_negative1",
"d_backup",
"d_garden",
"d_backup_negative2"
],
"candidate_order": [
"d_train",
"d_train_negative1",
"d_train_negative2",
"d_refund_negative1",
"d_refund",
"d_backup_negative1",
"d_refund_negative2",
"d_garden_negative2",
"d_garden_negative1",
"d_backup",
"d_garden",
"d_backup_negative2"
],
"reference_scores": [
0.9345145093980466,
0.9170717825497254,
0.911723885423255,
0.7317248664701558,
0.7513318377518619,
0.7041310675303637,
0.6939833258662486,
0.7105373505197093,
0.6719489263206282,
0.6797689367126396,
0.6965976234455196,
0.6990397887988085
],
"candidate_scores": [
0.9345144052082098,
0.9170717971357962,
0.9117239544823739,
0.7317249510183097,
0.7513317287155593,
0.7041310232862893,
0.6939833195844474,
0.7105374408265297,
0.6719490553333028,
0.6797688574510764,
0.6965976688094047,
0.6990399067735419
],
"reference_margin": 0.017442726848321155,
"candidate_margin": 0.0174426080724136,
"max_score_error": 1.2901267465448285e-07,
"top1_match": true,
"clear_pair_flips": 0
},
{
"query": "q_refund",
"reference_order": [
"d_refund",
"d_refund_negative2",
"d_refund_negative1",
"d_backup",
"d_train",
"d_train_negative1",
"d_train_negative2",
"d_backup_negative2",
"d_backup_negative1",
"d_garden",
"d_garden_negative2",
"d_garden_negative1"
],
"candidate_order": [
"d_refund",
"d_refund_negative2",
"d_refund_negative1",
"d_backup",
"d_train",
"d_train_negative1",
"d_train_negative2",
"d_backup_negative2",
"d_backup_negative1",
"d_garden",
"d_garden_negative2",
"d_garden_negative1"
],
"reference_scores": [
0.7214556217422334,
0.7144578286633229,
0.7132234614371428,
0.9581554437009046,
0.8592080211421944,
0.92480017781159,
0.7215637240319225,
0.67376318167836,
0.6822757076115306,
0.6410706404685216,
0.6339530415617187,
0.6408546761798151
],
"candidate_scores": [
0.7214555516319892,
0.7144578936624449,
0.7132236217713406,
0.9581554582142053,
0.8592079708488318,
0.9248001993191676,
0.7215637332380397,
0.6737632413580799,
0.682275776262881,
0.6410706355719538,
0.6339530681626802,
0.6408548397347046
],
"reference_margin": 0.03335526588931459,
"candidate_margin": 0.033355258895037765,
"max_score_error": 1.6355488952068953e-07,
"top1_match": true,
"clear_pair_flips": 0
},
{
"query": "q_backup",
"reference_order": [
"d_backup",
"d_backup_negative1",
"d_backup_negative2",
"d_refund",
"d_garden_negative2",
"d_garden",
"d_garden_negative1",
"d_train",
"d_train_negative2",
"d_train_negative1",
"d_refund_negative2",
"d_refund_negative1"
],
"candidate_order": [
"d_backup",
"d_backup_negative1",
"d_backup_negative2",
"d_refund",
"d_garden_negative2",
"d_garden",
"d_garden_negative1",
"d_train",
"d_train_negative2",
"d_train_negative1",
"d_refund_negative2",
"d_refund_negative1"
],
"reference_scores": [
0.7052823369965103,
0.6912889675495203,
0.6988308676272218,
0.7384987491405721,
0.6797166343077331,
0.6886572358164231,
0.9409091202419188,
0.8921029303277296,
0.8615508288157594,
0.7076615329689144,
0.7066142858726151,
0.7239074247176029
],
"candidate_scores": [
0.705282359983173,
0.6912890380654628,
0.6988310047944918,
0.7384988231067268,
0.679716571372278,
0.688657236922904,
0.9409091382959123,
0.8921029291014188,
0.8615508501458413,
0.7076615106019144,
0.7066143839072034,
0.7239075877883789
],
"reference_margin": 0.04880618991418917,
"candidate_margin": 0.04880620919449352,
"max_score_error": 1.630707759936456e-07,
"top1_match": true,
"clear_pair_flips": 0
},
{
"query": "q_garden",
"reference_order": [
"d_garden",
"d_garden_negative1",
"d_garden_negative2",
"d_backup_negative1",
"d_backup",
"d_train_negative2",
"d_train_negative1",
"d_train",
"d_backup_negative2",
"d_refund",
"d_refund_negative1",
"d_refund_negative2"
],
"candidate_order": [
"d_garden",
"d_garden_negative1",
"d_garden_negative2",
"d_backup_negative1",
"d_backup",
"d_train_negative2",
"d_train_negative1",
"d_train",
"d_backup_negative2",
"d_refund",
"d_refund_negative1",
"d_refund_negative2"
],
"reference_scores": [
0.660547675288807,
0.6813342545261133,
0.6865840756506723,
0.6282090252676062,
0.6272401859204846,
0.6238906236445231,
0.6881865592172072,
0.7191951945129387,
0.6567628481463305,
0.9618270896483002,
0.9089393242690795,
0.8270065201373913
],
"candidate_scores": [
0.660547697000132,
0.6813343524012313,
0.6865841838327507,
0.6282090650569285,
0.6272401486103947,
0.6238905531640648,
0.6881865024471118,
0.7191951750096339,
0.6567629254852003,
0.9618270311763162,
0.9089392935968987,
0.8270066728954764
],
"reference_margin": 0.052887765379220686,
"candidate_margin": 0.05288773757941745,
"max_score_error": 1.5275808507020372e-07,
"top1_match": true,
"clear_pair_flips": 0
}
],
"min_cosine": 0.9999999999988683,
"max_abs": 2.980232238769531e-07,
"wrong_pairing_control": {
"status": "FAIL",
"min_cosine": 0.6582763795695437
},
"runtime_record": {
"allowed_compute_units": [
"cpu",
"gpu",
"neuralEngine"
],
"artifact": "/Users/majimadaisuke/code/codex-conversions/2026-09-16/coreai-granite97m/exports/macos/granite97m_fp32_s512_bound.aimodel",
"artifact_verification": "PASS",
"backend": "gpu_preferred",
"backend_observation": "GPU preference; actual per-op placement not exposed by this runner",
"completed_fixture_count": 35,
"coreai_device_architecture": "h16c",
"current_fixture": "long_japanese",
"finished_at": "2026-09-19T06:17:58Z",
"fixture_sha256": "5ebed63dc08a8a7f3f5559d1a196e358c74ad6faa0222b6f638f6e88ae5957c3",
"gate_status": "NOT RUN: compare outputs with independent HF numerical and retrieval gate",
"load_cache_state": "not cleared; first load in this process only",
"load_ms": 469.995917,
"manifest_sha256": "1415cf22720c00a8f0000a27a343d01aabae8bfac573b28e122d4c05b7e45a76",
"model_sha": "835ad14087e140460703cf0fae09f97d469d65c2",
"os_version": "Version 27.0 (Build 26A428)",
"phase": "completed",
"precision": "fp32",
"preprocessing": "exact HF fixture input_ids and attention_mask; no tokenizer",
"requested_backend": "gpu",
"runner_binary_sha256": "a43ce2e2e4c6f855d785bcaef43a56114258de3514a80cfab56829867fb54d53",
"runner_schema_version": 1,
"runner_source_sha256": "30c0935d62f2f349a6a5e635d45b32d61abd518b53d6aec2d1a5876ce693718f",
"selected_fixture_count": 35,
"sequence_length": 512,
"started_at": "2026-09-19T06:17:55Z",
"status": "COMPUTED",
"total_fixture_count": 35,
"trial": 3
},
"input_hashes": {
"results/r01_mac_gpu_s512.json": "53522795574dee1cc402911bfa82308870291fb205bb15f39f9537317f226f35",
"fixtures/golden_s512.json": "5ebed63dc08a8a7f3f5559d1a196e358c74ad6faa0222b6f638f6e88ae5957c3",
"scripts/gate_metrics.py": "d4d73a64cb4f15850b1453ebe91beb4cfbf90056e8faaf10f5afb44991511c26",
"scripts/gate_returned.py": "d5605dbe65082207cf06516df2416117714cd874992c9870a71c434893509790"
}
}