| { |
| "accumulator_ceilings": { |
| "argued_i128_layernorm_bits": 126, |
| "i64_bits": 62 |
| }, |
| "batch": 16, |
| "exponents_sha256": "87c73b8f34990c8db9a9a48dbc66bf7b1f3502cea83913bc3679e8912ddd8ecb", |
| "format": "bob-width-sites-1", |
| "geometry": { |
| "ctx": 512, |
| "d_mlp": 1024, |
| "d_model": 256, |
| "head_dim": 32, |
| "n_head": 8, |
| "n_layer": 5, |
| "params": 5129216, |
| "positions": 8192, |
| "vocab": 4097 |
| }, |
| "model_config_digest": "4f7f5e807828cea50e906da035011b0d7203203fd6c18f4f654988d06c13598e", |
| "optimizer_schedule": { |
| "lr_den": 16, |
| "lr_num": 1, |
| "max_lr_den": 3200, |
| "max_lr_num": 200, |
| "max_tensor_lr_den": 26214400, |
| "max_tensor_lr_num": 104857600, |
| "optimizer": "sgd-momentum-7-8", |
| "schedule_total": 3128, |
| "warmup": 200 |
| }, |
| "sites": [ |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "token.common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "position.common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "embedding.checked_add", |
| "registered_bits": 24, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "residual_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| } |
| ], |
| "carrier": "checked i64 add, then i32", |
| "expression": "token_embedding + position_embedding after common rescale", |
| "id": "F01", |
| "name": "embed.add", |
| "proof": "token source bits=23 plus max(from_exp=-22-to_exp=-19,0)=0 -> 23 <= 62; position source bits=23 plus max(from_exp=-23-to_exp=-19,0)=0 -> 23 <= 62; checked add -> 24 <= 62; output act.resid.bits=23", |
| "rounding": "ties-away common rescale; no rounding at add", |
| "source": "src/net.rs::forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "mean_sum", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "sum(x_j), j<d_model", |
| "id": "F02", |
| "name": "ln.mean.sum", |
| "proof": "23 + ceil_log2(256)=8 -> 31 <= 62", |
| "rounding": "round-to-nearest, ties away on division by d_model", |
| "source": "src/net.rs::ln_forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "centered_value", |
| "registered_bits": 24, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| } |
| ], |
| "carrier": "checked i64 subtraction, then i32", |
| "expression": "x_j - mean", |
| "id": "F03", |
| "name": "ln.center", |
| "proof": "max(x=23, mean<2^23) + carry -> 24 <= 62", |
| "rounding": "none", |
| "source": "src/net.rs::ln_forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "square_sum", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "sum(center_j * center_j), j<d_model", |
| "id": "F04", |
| "name": "ln.square.sum", |
| "proof": "24 + 24 + ceil_log2(256)=8 -> 56 <= 62", |
| "rounding": "round-to-nearest, ties away on variance division", |
| "source": "src/net.rs::ln_forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "normalize_product", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "formula": "m_minus_59", |
| "kind": "runtime_ln_inv", |
| "m_source": "forward_saved_m", |
| "output_exponent": -24 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "normalized_q24", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -24 |
| } |
| } |
| ], |
| "carrier": "i64 product, then i32", |
| "expression": "round(center * inverse_std) into saved Q24 normalized value", |
| "id": "F05", |
| "name": "ln.normalize", |
| "proof": "product: 24 + 30 -> 54 <= 62; normalized_bits(d_model)=30", |
| "rounding": "round-to-nearest, ties away", |
| "source": "src/net.rs::ln_forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm1.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm1.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm1.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm1.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm1.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm1.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm1.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm1.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm1.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm1.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm1.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm1.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm1.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm1.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm1.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm1.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm1.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm1.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm1.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm1.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm1.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm1.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm1.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm1.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm1.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm2.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm2.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm2.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm2.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm2.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm2.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm2.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm2.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm2.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm2.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm2.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm2.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm2.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm2.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm2.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm2.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm2.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm2.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm2.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm2.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm2.natural_gamma_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm2.product_common", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm2.beta_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm2.checked_add", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm2.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| } |
| ], |
| "carrier": "i64 product/add, then i32", |
| "expression": "round(normalized * gamma) + beta after registered rescale", |
| "id": "F06", |
| "name": "ln.affine", |
| "proof": "final_norm product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-24 plus max(from_exp=-24-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17; layer_0.norm1 product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-28 plus max(from_exp=-28-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17; layer_1.norm1 product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-31 plus max(from_exp=-31-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17; layer_2.norm1 product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-31 plus max(from_exp=-31-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17; layer_3.norm1 product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-31 plus max(from_exp=-31-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17; layer_4.norm1 product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-31 plus max(from_exp=-31-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17; layer_0.norm2 product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-31 plus max(from_exp=-31-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17; layer_1.norm2 product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-31 plus max(from_exp=-31-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17; layer_2.norm2 product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-31 plus max(from_exp=-31-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17; layer_3.norm2 product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-31 plus max(from_exp=-31-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17; layer_4.norm2 product bits=53 exp=-43 plus max(from_exp=-43-to_exp=-17,0)=0 -> 53 <= 62; beta bits=23 exp=-30 plus max(from_exp=-30-to_exp=-17,0)=0 -> 23 <= 62; checked add -> 54 <= 62; output bits=23 exp=-17", |
| "rounding": "round-to-nearest, ties away at rescale", |
| "source": "src/net.rs::ln_forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.qkv.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.qkv.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.qkv.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.qkv.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.qkv.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.qkv.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.qkv.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.qkv.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.qkv.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.qkv.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.qkv.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.qkv.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.qkv.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.qkv.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.qkv.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.qkv.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.qkv.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.qkv.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.qkv.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.qkv.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.qkv.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.qkv.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.qkv.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.qkv.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.qkv.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "sum(ln1_j * qkv_weight_j), j<d_model", |
| "id": "F07", |
| "name": "qkv.matmul", |
| "proof": "23 + 23 + ceil_log2(256)=8 -> 54 <= 62; output act.qkv.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/net.rs::matmul_fwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "score_accumulator", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -36 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "sum(query_j * key_j), j<head_dim", |
| "id": "F08", |
| "name": "attention.score", |
| "proof": "23 + 23 + ceil_log2(32)=5 -> 51 <= 62", |
| "rounding": "none before score scaling", |
| "source": "src/net.rs::attention_forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "score_scale_product", |
| "registered_bits": 80, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -66 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "rounded_score_q26", |
| "registered_bits": 41, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| } |
| ], |
| "carrier": "argued i128 product, checked return to i64", |
| "expression": "round(attention_score * registered Q30 scale)", |
| "id": "F09", |
| "name": "score.scale", |
| "proof": "score_scale_q30=273841598 has mag_bits=29; non-accumulating wide product: 51 + 29 -> 80 <= 126", |
| "rounding": "round-to-nearest, ties away", |
| "source": "src/net.rs::attention_forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "inclusive_unsigned", |
| "name": "exp2_q30", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| } |
| ], |
| "carrier": "i64 inclusive unsigned Q30", |
| "expression": "registered exp2 table/interpolation output", |
| "id": "F10", |
| "name": "softmax.exp", |
| "proof": "registered table range is inclusive 0 <= e <= 2^30", |
| "rounding": "table interpolation uses registered integer rounding", |
| "source": "src/exp2.rs::exp2_q26" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "inclusive_unsigned", |
| "name": "exp_sum", |
| "registered_bits": 39, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator with inclusive upper endpoint", |
| "expression": "sum(exp_j) over a causal row", |
| "id": "F11", |
| "name": "softmax.sum", |
| "proof": "30 + ceil_log2(512)=9 -> 39 <= 62; exceptional inclusive law: 0 <= sum <= ctx*2^30", |
| "rounding": "none", |
| "source": "src/net.rs::attention_forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "inclusive_unsigned", |
| "name": "probability_q30", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| } |
| ], |
| "carrier": "i32 inclusive unsigned Q30", |
| "expression": "round((exp << 30) / row_sum)", |
| "id": "F12", |
| "name": "softmax.probability", |
| "proof": "exceptional inclusive law: 0 <= p <= 2^30, including the one-element row endpoint", |
| "rounding": "round-to-nearest, ties away", |
| "source": "src/net.rs::attention_forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "attention_value_accumulator", |
| "registered_bits": 62, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -48 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "attention_value_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator with inclusive Q30 broadcast operand", |
| "expression": "sum(probability_j * value_j) over causal row", |
| "id": "F13", |
| "name": "attention.value", |
| "proof": "30 + 23 + ceil_log2(512)=9 -> 62 <= 62; output act.attn_av.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/net.rs::attention_forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.attention_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.attention_projection.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.attention_projection.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.attention_projection.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.attention_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.attention_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.attention_projection.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.attention_projection.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.attention_projection.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.attention_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.attention_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -43 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.attention_projection.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.attention_projection.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.attention_projection.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.attention_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.attention_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.attention_projection.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.attention_projection.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.attention_projection.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.attention_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.attention_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.attention_projection.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.attention_projection.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.attention_projection.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.attention_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "attention value projection row-dot over d_model", |
| "id": "F14", |
| "name": "attention.output", |
| "proof": "23 + 23 + ceil_log2(256)=8 -> 54 <= 62; output act.attn_proj_out.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/net.rs::matmul_fwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_input.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_input.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_input.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_input.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_input.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_input.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_input.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_input.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_input.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_input.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_input.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_input.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_input.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_input.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_input.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_input.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_input.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_input.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_input.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_input.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_input.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_input.product_common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_input.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_input.checked_add", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_input.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "MLP input projection row-dot over d_model", |
| "id": "F15", |
| "name": "mlp.input", |
| "proof": "23 + 23 + ceil_log2(256)=8 -> 54 <= 62; output act.mlp_h.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/net.rs::matmul_fwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "slope_product", |
| "registered_bits": 34, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "rounded_activation", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| } |
| ], |
| "carrier": "i64 interpolation product, then i32", |
| "expression": "registered GELU table interpolation and rounded output", |
| "id": "F16", |
| "name": "activation.table", |
| "proof": "slope product: 14 + 20 -> 34 <= 62; TABLE_FRAC_BITS=14; registered max slope bits=20; output act.mlp_h.bits=23", |
| "rounding": "registered table ties-away rounding", |
| "source": "src/gelu.rs::gelu_site_at" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_output.accumulator", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_output.product_common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_output.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_output.checked_add", |
| "registered_bits": 57, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_output.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_output.accumulator", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_output.product_common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_output.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_output.checked_add", |
| "registered_bits": 57, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_output.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_output.accumulator", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_output.product_common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_output.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_output.checked_add", |
| "registered_bits": 57, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_output.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_output.accumulator", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_output.product_common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_output.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_output.checked_add", |
| "registered_bits": 57, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_output.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_output.accumulator", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_output.product_common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_output.bias_common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_output.checked_add", |
| "registered_bits": 57, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_output.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -21 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "MLP output projection row-dot over d_mlp", |
| "id": "F17", |
| "name": "mlp.output", |
| "proof": "23 + 23 + ceil_log2(1024)=10 -> 56 <= 62; output act.mlp_out.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/net.rs::matmul_fwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "residual.common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "attention.common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "mlp.common", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "residual.checked_add", |
| "registered_bits": 24, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "residual_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| } |
| ], |
| "carrier": "checked i64 add, then i32", |
| "expression": "residual + branch after common rescale", |
| "id": "F18", |
| "name": "residual.add", |
| "proof": "residual bits=23; attention source bits=23 plus max(from_exp=-22-to_exp=-19,0)=0 -> 23 <= 62; MLP source bits=23 plus max(from_exp=-21-to_exp=-19,0)=0 -> 23 <= 62; checked add -> 24 <= 62; output act.resid.bits=23", |
| "rounding": "ties-away common rescale; no rounding at add", |
| "source": "src/net.rs::forward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "logits_accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "logits_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -16 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "row-dot over d_model against tied token embedding", |
| "id": "F19", |
| "name": "logits", |
| "proof": "23 + 23 + ceil_log2(256)=8 -> 54 <= 62; output act.logits.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/net.rs::logits_and_softmax" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "logit_difference", |
| "registered_bits": 24, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -16 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "change_base_product", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -46 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "rounded_lookup_q26", |
| "registered_bits": 36, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| } |
| ], |
| "carrier": "i64 product", |
| "expression": "(logit - row_max) * LOG2E_Q30", |
| "id": "F20", |
| "name": "loss.change_base", |
| "proof": "difference bits=24; product: 24 + 31 -> 55 <= 62", |
| "rounding": "round-to-nearest, ties away into Q26", |
| "source": "src/net.rs::logits_and_softmax" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "loss_exp_sum", |
| "registered_bits": 43, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "sum(exp(logit - max)), j<vocab", |
| "id": "F21", |
| "name": "loss.exp_sum", |
| "proof": "30 + ceil_log2(4097)=13 -> 43 <= 62", |
| "rounding": "none", |
| "source": "src/net.rs::logits_and_softmax" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "inclusive_unsigned", |
| "name": "probability_at_d_logits", |
| "registered_bits": 20, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "d_logits_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| } |
| ], |
| "carrier": "checked i64 subtraction, then i32", |
| "expression": "computed probability - one_hot_target", |
| "id": "B01", |
| "name": "cross_entropy.gradient", |
| "proof": "inclusive Q30 probability plus subtraction bit; output bwd.d_logits.bits=23", |
| "rounding": "round-to-nearest, ties away in probability division", |
| "source": "src/backward.rs::ce_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "d_lnf_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -42 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "d_lnf_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "sum(dlogit_v * tied_weight_vj), v<vocab", |
| "id": "B02", |
| "name": "logits.input_gradient", |
| "proof": "23 + 23 + ceil_log2(4097)=13 -> 59 <= 62; output bwd.d_lnf_out.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::matmul_bwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "head.natural_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -37 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "embedding.natural_accumulator", |
| "registered_bits": 36, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "head.common", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -6 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "embedding.common", |
| "registered_bits": 36, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -6 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "branches.checked_add", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -6 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wte.gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -6 |
| } |
| } |
| ], |
| "carrier": "i64 accumulators and checked branch add", |
| "expression": "sum_n(dlogit_nv * final_norm_nj) plus embedding-path gradient", |
| "id": "B03", |
| "name": "tied_embedding.weight_gradient", |
| "proof": "head 23 + 23 + ceil_log2(8192)=13 -> 59 <= 62; embedding path 23 + ceil_log2(8192)=13 -> 36 <= 62; head common exponent source bits=59 plus max(from_exp=-37-to_exp=-6,0)=0 -> 59 <= 62; embedding common exponent source bits=36 plus max(from_exp=-14-to_exp=-6,0)=0 -> 36 <= 62; checked add -> 60 <= 62; wte grad.bits=23", |
| "rounding": "single ties-away gradient rescale", |
| "source": "src/backward.rs::backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| } |
| ], |
| "carrier": "i64 product, checked i32 local value", |
| "expression": "rescale(dy_j * gamma_j) to registered local g", |
| "id": "B04", |
| "name": "final_norm.local_g", |
| "proof": "local product: 23 + 23 -> 46 <= 62; narrowed dgamma accumulation: 23 + 24 + ceil_log2(8192)=13 -> 60 <= 62; output bwd.d_ln_g.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::ln_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "final_norm_gsum", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "sum(local_g_j), j<d_model", |
| "id": "B05", |
| "name": "final_norm.row_gsum", |
| "proof": "23 + ceil_log2(256)=8 -> 31 <= 62", |
| "rounding": "round-to-nearest, ties away on mean division", |
| "source": "src/backward.rs::ln_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "final_norm_bsum", |
| "registered_bits": 61, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| } |
| ], |
| "carrier": "argued i128 LayerNorm row accumulator", |
| "expression": "sum(local_g_j * normalized_j), j<d_model", |
| "id": "B06", |
| "name": "final_norm.row_bsum", |
| "proof": "23 + 30 + ceil_log2(256)=8 -> 61 <= 126", |
| "rounding": "round-to-nearest, ties away on mean division", |
| "source": "src/backward.rs::ln_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.normalized_bmean_product", |
| "registered_bits": 83, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -68 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.post_q48_projection", |
| "registered_bits": 28, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.three_term", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.inverse_product", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "formula": "m_plus_g_minus_x_minus_59", |
| "kind": "runtime_ln_inv", |
| "m_source": "final_norm_saved_m", |
| "output_exponent": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "final_norm.dx", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| } |
| ], |
| "carrier": "argued i128 intermediate, checked i64 terms, then i32", |
| "expression": "normalized-gradient formula for final norm input", |
| "id": "B07", |
| "name": "final_norm.dx", |
| "proof": "normalized*bmean needs 83 <= 126; runtime normalized-energy premise sum(n^2)<=d*(2^24+1)^2; post-Q48 projection needs d_ln_g.bits=23 + conservative sqrt-width=5 -> 28 <= 62; three-term subtraction needs 30 <= 62; final term*inverse needs 60 <= 62; output bwd.d_resid.bits=23", |
| "rounding": "round-to-nearest, ties away", |
| "source": "src/backward.rs::ln_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_projection.common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_projection.common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_projection.common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_projection.common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_projection.common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "dY @ W^T, inner width d_model", |
| "id": "B08", |
| "name": "mlp_output.input_gradient", |
| "proof": "23 + 23 + ceil_log2(256)=8 -> 54 <= 62; output bwd.d_mlp_h.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::matmul_bwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_projection.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -35 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_projection.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_projection.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -35 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_projection.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_projection.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -35 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_projection.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_projection.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -35 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_projection.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_projection.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -35 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_projection.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "X^T @ dY, inner width batch*ctx", |
| "id": "B09", |
| "name": "mlp_output.weight_gradient", |
| "proof": "23 + 23 + ceil_log2(8192)=13 -> 59 <= 62; max grad.bits=23", |
| "rounding": "single ties-away gradient rescale", |
| "source": "src/backward.rs::matmul_bwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "gelu_derivative_q29", |
| "name": "gelu_derivative_q29", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -29 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "derivative_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -48 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "rounded_d_mlp_h", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| } |
| ], |
| "carrier": "i64 product, then i32", |
| "expression": "dY * registered GELU derivative", |
| "id": "B10", |
| "name": "activation.derivative", |
| "proof": "23 + 30 -> 53 <= 62; output bwd.d_mlp_h.bits=23", |
| "rounding": "registered table and ties-away product rounding", |
| "source": "src/gelu.rs::gelu_backward_at_site" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_input.accumulator", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -42 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_input.common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_input.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_input.accumulator", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -42 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_input.common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_input.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_input.accumulator", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -42 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_input.common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_input.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_input.accumulator", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -42 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_input.common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_input.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_input.accumulator", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -42 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_input.common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_input.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "dH @ W^T, inner width d_mlp", |
| "id": "B11", |
| "name": "mlp_input.input_gradient", |
| "proof": "23 + 23 + ceil_log2(1024)=10 -> 56 <= 62; output bwd.d_ln2_out.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::matmul_bwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_input.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -36 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.mlp_input.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_input.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -36 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.mlp_input.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_input.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -36 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.mlp_input.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_input.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -36 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.mlp_input.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_input.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -36 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.mlp_input.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "X^T @ dH, inner width batch*ctx", |
| "id": "B12", |
| "name": "mlp_input.weight_gradient", |
| "proof": "23 + 23 + ceil_log2(8192)=13 -> 59 <= 62; max grad.bits=23", |
| "rounding": "single ties-away gradient rescale", |
| "source": "src/backward.rs::matmul_bwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm2.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm2.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm2.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm2.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm2.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm2.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm2.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm2.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm2.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm2.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm2.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm2.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm2.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm2.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm2.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm2.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm2.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm2.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm2.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm2.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| } |
| ], |
| "carrier": "i64 product, checked i32 local value", |
| "expression": "rescale(dy_j * gamma_j) to registered local g", |
| "id": "B13", |
| "name": "norm2.local_g", |
| "proof": "local product: 23 + 23 -> 46 <= 62; narrowed dgamma accumulation: 23 + 24 + ceil_log2(8192)=13 -> 60 <= 62; output bwd.d_ln_g.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::ln_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "norm2.gsum", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "norm2.bsum", |
| "registered_bits": 61, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| } |
| ], |
| "carrier": "i64 gsum and argued i128 LayerNorm bsum", |
| "expression": "sum(g) and sum(g * normalized), j<d_model", |
| "id": "B14", |
| "name": "norm2.row_sums", |
| "proof": "gsum 23 + ceil_log2(256)=8 -> 31 <= 62; bsum 23 + 30 + ceil_log2(256)=8 -> 61 <= 126", |
| "rounding": "round-to-nearest, ties away on division", |
| "source": "src/backward.rs::ln_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "norm2.normalized_bmean_product", |
| "registered_bits": 83, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -68 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "norm2.post_q48_projection", |
| "registered_bits": 28, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "norm2.three_term", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "norm2.inverse_product", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "formula": "m_plus_g_minus_x_minus_59", |
| "kind": "runtime_ln_inv", |
| "m_source": "norm2_saved_m", |
| "output_exponent": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "norm2.dx", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| } |
| ], |
| "carrier": "argued i128 intermediate, checked i64 terms, then i32", |
| "expression": "normalized-gradient formula before residual-gradient add", |
| "id": "B15", |
| "name": "norm2.dx", |
| "proof": "normalized*bmean 83 <= 126; runtime normalized-energy premise sum(n^2)<=d*(2^24+1)^2; post-Q48 projection 28 <= 62; three-term subtraction 30 <= 62; final product 60 <= 62; output bwd.d_resid.bits=23", |
| "rounding": "round-to-nearest, ties away", |
| "source": "src/backward.rs::ln_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.attention_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.attention_projection.common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.attention_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.attention_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.attention_projection.common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.attention_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.attention_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.attention_projection.common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.attention_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.attention_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.attention_projection.common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.attention_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.attention_projection.accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.attention_projection.common", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.attention_projection.output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "dY @ W^T, inner width d_model", |
| "id": "B16", |
| "name": "attention_output.input_gradient", |
| "proof": "23 + 23 + ceil_log2(256)=8 -> 54 <= 62; output bwd.d_attn_av.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::matmul_bwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.attention_projection.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -34 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.attention_projection.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.attention_projection.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -34 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.attention_projection.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.attention_projection.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -34 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.attention_projection.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.attention_projection.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -34 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.attention_projection.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.attention_projection.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -34 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.attention_projection.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "X^T @ dY, inner width batch*ctx", |
| "id": "B17", |
| "name": "attention_output.weight_gradient", |
| "proof": "23 + 23 + ceil_log2(8192)=13 -> 59 <= 62; max grad.bits=23", |
| "rounding": "single ties-away gradient rescale", |
| "source": "src/backward.rs::matmul_bwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "value_gradient_accumulator", |
| "registered_bits": 62, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -48 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "d_qkv_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator with inclusive Q30 broadcast operand", |
| "expression": "sum_i(probability_ij * dA_iv), causal extent", |
| "id": "B18", |
| "name": "value.gradient", |
| "proof": "30 + 23 + ceil_log2(512)=9 -> 62 <= 62; output bwd.d_qkv.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::attention_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "probability_gradient_accumulator", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -36 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "d_p_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "dA_i dot V_j over head_dim", |
| "id": "B19", |
| "name": "probability.gradient", |
| "proof": "23 + 23 + ceil_log2(32)=5 -> 51 <= 62; output bwd.d_p.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::attention_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "softmax_rowdot_accumulator", |
| "registered_bits": 62, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -47 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator with inclusive Q30 operand", |
| "expression": "sum_j(dP_j * probability_j) over causal row", |
| "id": "B20", |
| "name": "softmax.rowdot", |
| "proof": "23 + 30 + ceil_log2(512)=9 -> 62 <= 62", |
| "rounding": "round-to-nearest, ties away to row mean", |
| "source": "src/backward.rs::attention_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "softmax_subtraction", |
| "registered_bits": 24, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "softmax_product", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -47 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "rounded_softmax_gradient", |
| "registered_bits": 24, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| } |
| ], |
| "carrier": "i64 product, then checked value", |
| "expression": "probability_j * (dP_j - rowdot_mean)", |
| "id": "B21", |
| "name": "softmax.gradient", |
| "proof": "subtraction bits=24; product 30 + 24 -> 54 <= 62", |
| "rounding": "round-to-nearest, ties away by Q30", |
| "source": "src/backward.rs::attention_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "rsqrt_product", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -47 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "rounded_score_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -17 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "d_score_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| } |
| ], |
| "carrier": "i64 product, then i32", |
| "expression": "softmax gradient * registered reciprocal-sqrt Q30 scale", |
| "id": "B22", |
| "name": "score.unscale", |
| "proof": "rsqrt_head_dim_q30=189812531 has mag_bits=28; 24 + 28 -> 52 <= 62; output bwd.d_score.bits=23", |
| "rounding": "round-to-nearest, ties away by Q30 and site rescale", |
| "source": "src/backward.rs::attention_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "query_gradient_accumulator", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "d_qkv_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "dScore @ K over causal context extent", |
| "id": "B23", |
| "name": "query.gradient", |
| "proof": "23 + 23 + ceil_log2(512)=9 -> 55 <= 62; output bwd.d_qkv.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::attention_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "key_gradient_accumulator", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -40 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "d_qkv_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -18 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "dScore^T @ Q over causal context extent", |
| "id": "B24", |
| "name": "key.gradient", |
| "proof": "23 + 23 + ceil_log2(512)=9 -> 55 <= 62; output bwd.d_qkv.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::attention_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.branch_accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.three_branch_add", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.combined_common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.d_ln1_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.branch_accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.three_branch_add", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.combined_common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.d_ln1_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.branch_accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.three_branch_add", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.combined_common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.d_ln1_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.branch_accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.three_branch_add", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.combined_common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.d_ln1_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.branch_accumulator", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.three_branch_add", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -41 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.combined_common", |
| "registered_bits": 56, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.d_ln1_output", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "combined Q/K/V gradient times fused projection weights", |
| "id": "B25", |
| "name": "qkv.input_gradient", |
| "proof": "each Q/K/V branch 23 + 23 + ceil_log2(256)=8 -> 54 <= 62; fixed three-branch add needs ceil_log2(3)=2 -> 56 <= 62; output bwd.d_ln1_out.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::matmul_bwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.qkv.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -35 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.qkv.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.qkv.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -35 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.qkv.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.qkv.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -35 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.qkv.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.qkv.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -35 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.qkv.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.qkv.gradient_accumulator", |
| "registered_bits": 59, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -35 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.qkv.weight_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| } |
| ], |
| "carrier": "i64 accumulator", |
| "expression": "X^T @ dQKV, inner width batch*ctx", |
| "id": "B26", |
| "name": "qkv.weight_gradient", |
| "proof": "23 + 23 + ceil_log2(8192)=13 -> 59 <= 62; max grad.bits=23", |
| "rounding": "single ties-away gradient rescale", |
| "source": "src/backward.rs::matmul_bwd" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm1.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm1.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm1.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_0.norm1.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm1.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm1.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm1.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_1.norm1.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm1.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm1.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm1.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_2.norm1.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm1.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm1.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm1.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_3.norm1.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm1.local_g_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -39 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm1.local_g", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm1.dgamma_accumulator", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -38 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "layer_4.norm1.gamma_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| } |
| ], |
| "carrier": "i64 product, checked i32 local value", |
| "expression": "rescale(dy_j * gamma_j) to registered local g", |
| "id": "B27", |
| "name": "norm1.local_g", |
| "proof": "local product: 23 + 23 -> 46 <= 62; narrowed dgamma accumulation: 23 + 24 + ceil_log2(8192)=13 -> 60 <= 62; output bwd.d_ln_g.bits=23", |
| "rounding": "single ties-away rescale", |
| "source": "src/backward.rs::ln_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "norm1.gsum", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "norm1.bsum", |
| "registered_bits": 61, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -44 |
| } |
| } |
| ], |
| "carrier": "i64 gsum and argued i128 LayerNorm bsum", |
| "expression": "sum(g) and sum(g * normalized), j<d_model", |
| "id": "B28", |
| "name": "norm1.row_sums", |
| "proof": "gsum 23 + ceil_log2(256)=8 -> 31 <= 62; bsum 23 + 30 + ceil_log2(256)=8 -> 61 <= 126", |
| "rounding": "round-to-nearest, ties away on division", |
| "source": "src/backward.rs::ln_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "norm1.normalized_bmean_product", |
| "registered_bits": 83, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -68 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "norm1.post_q48_projection", |
| "registered_bits": 28, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "norm1.three_term", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -20 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "norm1.inverse_product", |
| "registered_bits": 60, |
| "registered_exponent": { |
| "formula": "m_plus_g_minus_x_minus_59", |
| "kind": "runtime_ln_inv", |
| "m_source": "norm1_saved_m", |
| "output_exponent": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "norm1.dx", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| } |
| ], |
| "carrier": "argued i128 intermediate, checked i64 terms, then i32", |
| "expression": "normalized-gradient formula before embedding-gradient add", |
| "id": "B29", |
| "name": "norm1.dx", |
| "proof": "normalized*bmean 83 <= 126; runtime normalized-energy premise sum(n^2)<=d*(2^24+1)^2; post-Q48 projection 28 <= 62; three-term subtraction 30 <= 62; final product 60 <= 62; output bwd.d_resid.bits=23", |
| "rounding": "round-to-nearest, ties away", |
| "source": "src/backward.rs::ln_backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "token.accumulator", |
| "registered_bits": 36, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "position.accumulator", |
| "registered_bits": 36, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "position.wpe_gradient", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| } |
| ], |
| "carrier": "two disjoint i64 accumulators", |
| "expression": "token and position path gradient accumulation at disjoint destinations", |
| "id": "B30", |
| "name": "embedding.gradient_add", |
| "proof": "token 23 + ceil_log2(8192)=13 -> 36 <= 62; position 23 + ceil_log2(8192)=13 -> 36 <= 62; disjoint destinations, max -> 36 <= 62; max grad.bits=23", |
| "rounding": "single ties-away gradient rescale", |
| "source": "src/backward.rs::backward" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -7 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -7 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -7 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -7 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -7 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -7 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -14 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -13 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -8 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -9 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.bias.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.bias.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.bias.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -12 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.weight.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.weight.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.weight.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -10 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wpe.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wpe.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wpe.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -11 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wte.multiplier_product", |
| "registered_bits": 30, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -6 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wte.raw_momentum_add", |
| "registered_bits": 31, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -6 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wte.momentum", |
| "registered_bits": 27, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -6 |
| } |
| } |
| ], |
| "carrier": "i64 product/add and registered i64 state", |
| "expression": "round((7 * momentum + gradient) / 8)", |
| "id": "B31", |
| "name": "momentum.update", |
| "proof": "max(m_bits=27+mag_bits(7)=3, grad.bits=23)+1 -> 31 <= 62; output adam.m_bits=27", |
| "rounding": "round-to-nearest, ties away by 2^3", |
| "source": "src/optim.rs::sgd_step_at_sites" |
| }, |
| { |
| "boundaries": [ |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.weight.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.weight.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.proj.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.bias.lr_product", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -27 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.bias.subtraction", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -27 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -27 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.weight.lr_product", |
| "registered_bits": 48, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.weight.subtraction", |
| "registered_bits": 49, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.attn.qkv.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.bias.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -28 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.bias.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -28 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -28 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.weight.lr_product", |
| "registered_bits": 42, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.weight.subtraction", |
| "registered_bits": 43, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln1.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.bias.lr_product", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.bias.subtraction", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.weight.lr_product", |
| "registered_bits": 41, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.weight.subtraction", |
| "registered_bits": 42, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.ln2.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -29 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -29 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -29 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.weight.lr_product", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.weight.subtraction", |
| "registered_bits": 48, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.fc.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.bias.lr_product", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.bias.subtraction", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.weight.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.weight.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.0.mlp.proj.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.weight.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.weight.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.proj.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -29 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -29 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -29 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.weight.lr_product", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.weight.subtraction", |
| "registered_bits": 48, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.attn.qkv.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.weight.lr_product", |
| "registered_bits": 41, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.weight.subtraction", |
| "registered_bits": 42, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln1.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.weight.lr_product", |
| "registered_bits": 41, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.weight.subtraction", |
| "registered_bits": 42, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.ln2.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.weight.lr_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.weight.subtraction", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.fc.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.weight.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.weight.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.1.mlp.proj.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.weight.lr_product", |
| "registered_bits": 50, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -24 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.weight.subtraction", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -24 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.proj.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -24 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.bias.lr_product", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.bias.subtraction", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.weight.lr_product", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.weight.subtraction", |
| "registered_bits": 48, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.attn.qkv.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.weight.lr_product", |
| "registered_bits": 40, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.weight.subtraction", |
| "registered_bits": 41, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln1.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.weight.lr_product", |
| "registered_bits": 41, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.weight.subtraction", |
| "registered_bits": 42, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.ln2.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.weight.lr_product", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.weight.subtraction", |
| "registered_bits": 48, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.fc.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.weight.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.weight.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.2.mlp.proj.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.weight.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.weight.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.proj.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.bias.lr_product", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.bias.subtraction", |
| "registered_bits": 55, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.weight.lr_product", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.weight.subtraction", |
| "registered_bits": 48, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.attn.qkv.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.weight.lr_product", |
| "registered_bits": 40, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.weight.subtraction", |
| "registered_bits": 41, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln1.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.weight.lr_product", |
| "registered_bits": 41, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.weight.subtraction", |
| "registered_bits": 42, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.ln2.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.weight.lr_product", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.weight.subtraction", |
| "registered_bits": 48, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.fc.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.weight.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.weight.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.3.mlp.proj.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.weight.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.weight.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.proj.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.bias.lr_product", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -29 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.bias.subtraction", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -29 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -29 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.weight.lr_product", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.weight.subtraction", |
| "registered_bits": 48, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.attn.qkv.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -31 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.weight.lr_product", |
| "registered_bits": 41, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.weight.subtraction", |
| "registered_bits": 42, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln1.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.bias.lr_product", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.bias.subtraction", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.weight.lr_product", |
| "registered_bits": 41, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.weight.subtraction", |
| "registered_bits": 42, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.ln2.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.bias.lr_product", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.bias.subtraction", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -30 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.weight.lr_product", |
| "registered_bits": 46, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.weight.subtraction", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.fc.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.bias.lr_product", |
| "registered_bits": 53, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.bias.subtraction", |
| "registered_bits": 54, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -26 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.weight.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.weight.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "h.4.mlp.proj.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -25 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.bias.lr_product", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -24 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.bias.subtraction", |
| "registered_bits": 48, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -24 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.bias.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -24 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.weight.lr_product", |
| "registered_bits": 44, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.weight.subtraction", |
| "registered_bits": 45, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "ln_f.weight.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -19 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wpe.lr_product", |
| "registered_bits": 47, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wpe.subtraction", |
| "registered_bits": 48, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wpe.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -23 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wte.lr_product", |
| "registered_bits": 51, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wte.subtraction", |
| "registered_bits": 52, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| }, |
| { |
| "interval": "strict_signed", |
| "name": "wte.weight", |
| "registered_bits": 23, |
| "registered_exponent": { |
| "kind": "fixed", |
| "value": -22 |
| } |
| } |
| ], |
| "carrier": "checked i64 product/subtraction, then i32", |
| "expression": "weight - round(lr_num * momentum / lr_den)", |
| "id": "B32", |
| "name": "weight.update", |
| "proof": "optimizer=sgd-momentum-7-8; h.0.attn.proj.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.0.attn.proj.weight tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23; h.0.attn.qkv.bias tensor numerator=26214400 needs ceil_log2(constant)=25; product with momentum.bits=27 -> 52 <= 62; subtraction max(weight.bits=23,delta.bits=52)+1 -> 53 <= 62; output weight.bits=23; h.0.attn.qkv.weight tensor numerator=1638400 needs ceil_log2(constant)=21; product with momentum.bits=27 -> 48 <= 62; subtraction max(weight.bits=23,delta.bits=48)+1 -> 49 <= 62; output weight.bits=23; h.0.ln1.bias tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23; h.0.ln1.weight tensor numerator=25600 needs ceil_log2(constant)=15; product with momentum.bits=27 -> 42 <= 62; subtraction max(weight.bits=23,delta.bits=42)+1 -> 43 <= 62; output weight.bits=23; h.0.ln2.bias tensor numerator=104857600 needs ceil_log2(constant)=27; product with momentum.bits=27 -> 54 <= 62; subtraction max(weight.bits=23,delta.bits=54)+1 -> 55 <= 62; output weight.bits=23; h.0.ln2.weight tensor numerator=12800 needs ceil_log2(constant)=14; product with momentum.bits=27 -> 41 <= 62; subtraction max(weight.bits=23,delta.bits=41)+1 -> 42 <= 62; output weight.bits=23; h.0.mlp.fc.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.0.mlp.fc.weight tensor numerator=819200 needs ceil_log2(constant)=20; product with momentum.bits=27 -> 47 <= 62; subtraction max(weight.bits=23,delta.bits=47)+1 -> 48 <= 62; output weight.bits=23; h.0.mlp.proj.bias tensor numerator=104857600 needs ceil_log2(constant)=27; product with momentum.bits=27 -> 54 <= 62; subtraction max(weight.bits=23,delta.bits=54)+1 -> 55 <= 62; output weight.bits=23; h.0.mlp.proj.weight tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23; h.1.attn.proj.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.1.attn.proj.weight tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23; h.1.attn.qkv.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.1.attn.qkv.weight tensor numerator=819200 needs ceil_log2(constant)=20; product with momentum.bits=27 -> 47 <= 62; subtraction max(weight.bits=23,delta.bits=47)+1 -> 48 <= 62; output weight.bits=23; h.1.ln1.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.1.ln1.weight tensor numerator=12800 needs ceil_log2(constant)=14; product with momentum.bits=27 -> 41 <= 62; subtraction max(weight.bits=23,delta.bits=41)+1 -> 42 <= 62; output weight.bits=23; h.1.ln2.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.1.ln2.weight tensor numerator=12800 needs ceil_log2(constant)=14; product with momentum.bits=27 -> 41 <= 62; subtraction max(weight.bits=23,delta.bits=41)+1 -> 42 <= 62; output weight.bits=23; h.1.mlp.fc.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.1.mlp.fc.weight tensor numerator=409600 needs ceil_log2(constant)=19; product with momentum.bits=27 -> 46 <= 62; subtraction max(weight.bits=23,delta.bits=46)+1 -> 47 <= 62; output weight.bits=23; h.1.mlp.proj.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.1.mlp.proj.weight tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23; h.2.attn.proj.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.2.attn.proj.weight tensor numerator=6553600 needs ceil_log2(constant)=23; product with momentum.bits=27 -> 50 <= 62; subtraction max(weight.bits=23,delta.bits=50)+1 -> 51 <= 62; output weight.bits=23; h.2.attn.qkv.bias tensor numerator=104857600 needs ceil_log2(constant)=27; product with momentum.bits=27 -> 54 <= 62; subtraction max(weight.bits=23,delta.bits=54)+1 -> 55 <= 62; output weight.bits=23; h.2.attn.qkv.weight tensor numerator=819200 needs ceil_log2(constant)=20; product with momentum.bits=27 -> 47 <= 62; subtraction max(weight.bits=23,delta.bits=47)+1 -> 48 <= 62; output weight.bits=23; h.2.ln1.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.2.ln1.weight tensor numerator=6400 needs ceil_log2(constant)=13; product with momentum.bits=27 -> 40 <= 62; subtraction max(weight.bits=23,delta.bits=40)+1 -> 41 <= 62; output weight.bits=23; h.2.ln2.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.2.ln2.weight tensor numerator=12800 needs ceil_log2(constant)=14; product with momentum.bits=27 -> 41 <= 62; subtraction max(weight.bits=23,delta.bits=41)+1 -> 42 <= 62; output weight.bits=23; h.2.mlp.fc.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.2.mlp.fc.weight tensor numerator=819200 needs ceil_log2(constant)=20; product with momentum.bits=27 -> 47 <= 62; subtraction max(weight.bits=23,delta.bits=47)+1 -> 48 <= 62; output weight.bits=23; h.2.mlp.proj.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.2.mlp.proj.weight tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23; h.3.attn.proj.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.3.attn.proj.weight tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23; h.3.attn.qkv.bias tensor numerator=104857600 needs ceil_log2(constant)=27; product with momentum.bits=27 -> 54 <= 62; subtraction max(weight.bits=23,delta.bits=54)+1 -> 55 <= 62; output weight.bits=23; h.3.attn.qkv.weight tensor numerator=819200 needs ceil_log2(constant)=20; product with momentum.bits=27 -> 47 <= 62; subtraction max(weight.bits=23,delta.bits=47)+1 -> 48 <= 62; output weight.bits=23; h.3.ln1.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.3.ln1.weight tensor numerator=6400 needs ceil_log2(constant)=13; product with momentum.bits=27 -> 40 <= 62; subtraction max(weight.bits=23,delta.bits=40)+1 -> 41 <= 62; output weight.bits=23; h.3.ln2.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.3.ln2.weight tensor numerator=12800 needs ceil_log2(constant)=14; product with momentum.bits=27 -> 41 <= 62; subtraction max(weight.bits=23,delta.bits=41)+1 -> 42 <= 62; output weight.bits=23; h.3.mlp.fc.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.3.mlp.fc.weight tensor numerator=819200 needs ceil_log2(constant)=20; product with momentum.bits=27 -> 47 <= 62; subtraction max(weight.bits=23,delta.bits=47)+1 -> 48 <= 62; output weight.bits=23; h.3.mlp.proj.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.3.mlp.proj.weight tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23; h.4.attn.proj.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.4.attn.proj.weight tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23; h.4.attn.qkv.bias tensor numerator=26214400 needs ceil_log2(constant)=25; product with momentum.bits=27 -> 52 <= 62; subtraction max(weight.bits=23,delta.bits=52)+1 -> 53 <= 62; output weight.bits=23; h.4.attn.qkv.weight tensor numerator=819200 needs ceil_log2(constant)=20; product with momentum.bits=27 -> 47 <= 62; subtraction max(weight.bits=23,delta.bits=47)+1 -> 48 <= 62; output weight.bits=23; h.4.ln1.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.4.ln1.weight tensor numerator=12800 needs ceil_log2(constant)=14; product with momentum.bits=27 -> 41 <= 62; subtraction max(weight.bits=23,delta.bits=41)+1 -> 42 <= 62; output weight.bits=23; h.4.ln2.bias tensor numerator=26214400 needs ceil_log2(constant)=25; product with momentum.bits=27 -> 52 <= 62; subtraction max(weight.bits=23,delta.bits=52)+1 -> 53 <= 62; output weight.bits=23; h.4.ln2.weight tensor numerator=12800 needs ceil_log2(constant)=14; product with momentum.bits=27 -> 41 <= 62; subtraction max(weight.bits=23,delta.bits=41)+1 -> 42 <= 62; output weight.bits=23; h.4.mlp.fc.bias tensor numerator=26214400 needs ceil_log2(constant)=25; product with momentum.bits=27 -> 52 <= 62; subtraction max(weight.bits=23,delta.bits=52)+1 -> 53 <= 62; output weight.bits=23; h.4.mlp.fc.weight tensor numerator=409600 needs ceil_log2(constant)=19; product with momentum.bits=27 -> 46 <= 62; subtraction max(weight.bits=23,delta.bits=46)+1 -> 47 <= 62; output weight.bits=23; h.4.mlp.proj.bias tensor numerator=52428800 needs ceil_log2(constant)=26; product with momentum.bits=27 -> 53 <= 62; subtraction max(weight.bits=23,delta.bits=53)+1 -> 54 <= 62; output weight.bits=23; h.4.mlp.proj.weight tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23; ln_f.bias tensor numerator=819200 needs ceil_log2(constant)=20; product with momentum.bits=27 -> 47 <= 62; subtraction max(weight.bits=23,delta.bits=47)+1 -> 48 <= 62; output weight.bits=23; ln_f.weight tensor numerator=102400 needs ceil_log2(constant)=17; product with momentum.bits=27 -> 44 <= 62; subtraction max(weight.bits=23,delta.bits=44)+1 -> 45 <= 62; output weight.bits=23; wpe tensor numerator=819200 needs ceil_log2(constant)=20; product with momentum.bits=27 -> 47 <= 62; subtraction max(weight.bits=23,delta.bits=47)+1 -> 48 <= 62; output weight.bits=23; wte tensor numerator=13107200 needs ceil_log2(constant)=24; product with momentum.bits=27 -> 51 <= 62; subtraction max(weight.bits=23,delta.bits=51)+1 -> 52 <= 62; output weight.bits=23", |
| "rounding": "round-to-nearest, ties away", |
| "source": "src/optim.rs::sgd_step_at_sites" |
| } |
| ] |
| } |
|
|
