registration/exponents.json

{
  "act": {
    "attn_av": {
      "bits": 23,
      "exp": -19
    },
    "attn_proj_out": {
      "bits": 23,
      "exp": -22
    },
    "ln1_out": {
      "bits": 23,
      "exp": -17
    },
    "ln2_out": {
      "bits": 23,
      "exp": -17
    },
    "lnf_out": {
      "bits": 23,
      "exp": -17
    },
    "logits": {
      "bits": 23,
      "exp": -16
    },
    "mlp_h": {
      "bits": 23,
      "exp": -19
    },
    "mlp_out": {
      "bits": 23,
      "exp": -21
    },
    "qkv": {
      "bits": 23,
      "exp": -18
    },
    "resid": {
      "bits": 23,
      "exp": -19
    }
  },
  "adam": {
    "eps_int": 131072,
    "m_bits": 27,
    "sqrt_shift": 7,
    "v_bits": 48
  },
  "backoff_bits": 0,
  "bwd": {
    "d_attn_av": {
      "bits": 23,
      "exp": -18
    },
    "d_attn_proj_out": {
      "bits": 23,
      "exp": -15
    },
    "d_ln1_out": {
      "bits": 23,
      "exp": -20
    },
    "d_ln2_out": {
      "bits": 23,
      "exp": -20
    },
    "d_ln_g": {
      "bits": 23,
      "exp": -20
    },
    "d_lnf_out": {
      "bits": 23,
      "exp": -22
    },
    "d_logits": {
      "bits": 23,
      "exp": -20
    },
    "d_mlp_h": {
      "bits": 23,
      "exp": -19
    },
    "d_mlp_out": {
      "bits": 23,
      "exp": -16
    },
    "d_p": {
      "bits": 23,
      "exp": -17
    },
    "d_qkv": {
      "bits": 23,
      "exp": -18
    },
    "d_resid": {
      "bits": 23,
      "exp": -14
    },
    "d_score": {
      "bits": 23,
      "exp": -22
    }
  },
  "eps_raw": [
    2748779,
    2748779,
    2748779,
    2748779,
    2748779,
    2748779,
    2748779,
    2748779,
    2748779,
    2748779,
    2748779
  ],
  "format": "bob-int-exponents-1",
  "grad": [
    {
      "bits": 23,
      "exp": -7
    },
    {
      "bits": 23,
      "exp": -9
    },
    {
      "bits": 23,
      "exp": -10
    },
    {
      "bits": 23,
      "exp": -10
    },
    {
      "bits": 23,
      "exp": -12
    },
    {
      "bits": 23,
      "exp": -12
    },
    {
      "bits": 23,
      "exp": -12
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -7
    },
    {
      "bits": 23,
      "exp": -9
    },
    {
      "bits": 23,
      "exp": -8
    },
    {
      "bits": 23,
      "exp": -9
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -12
    },
    {
      "bits": 23,
      "exp": -12
    },
    {
      "bits": 23,
      "exp": -8
    },
    {
      "bits": 23,
      "exp": -9
    },
    {
      "bits": 23,
      "exp": -8
    },
    {
      "bits": 23,
      "exp": -9
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -14
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -12
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -8
    },
    {
      "bits": 23,
      "exp": -9
    },
    {
      "bits": 23,
      "exp": -8
    },
    {
      "bits": 23,
      "exp": -9
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -14
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -12
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -8
    },
    {
      "bits": 23,
      "exp": -9
    },
    {
      "bits": 23,
      "exp": -8
    },
    {
      "bits": 23,
      "exp": -9
    },
    {
      "bits": 23,
      "exp": -12
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -13
    },
    {
      "bits": 23,
      "exp": -12
    },
    {
      "bits": 23,
      "exp": -8
    },
    {
      "bits": 23,
      "exp": -9
    },
    {
      "bits": 23,
      "exp": -12
    },
    {
      "bits": 23,
      "exp": -10
    },
    {
      "bits": 23,
      "exp": -11
    },
    {
      "bits": 23,
      "exp": -6
    }
  ],
  "loss_exp": -20,
  "note": "geometry-derived carrier targets: weight=20 activation=20 gradient=20; inclusive-Q30 attention row at ctx=512 closes at 62 of 62 accumulator bits; shadow_contract={\"batch\":16,\"config_path\":\"/Volumes/F32WORK/training-scratch/bob/mvc/software/internal/ml/bob/train-int/mvc/config.json\",\"config_sha256\":\"79bb7297951e9667305ebac3498d89206bf8b7cf22a1d7169009dd0bdc337762\",\"config_source_sha256\":\"79bb7297951e9667305ebac3498d89206bf8b7cf22a1d7169009dd0bdc337762\",\"corpus_path\":\"/Volumes/F32WORK/training-scratch/bob/mvc/software/internal/ml/detcorpus/freeze/streams/all/tokens.u16.le.bin\",\"corpus_sha256\":\"5b7343b6415ce15dafba5ce951c3da546be64d28f8124a5a6fbea22196630502\",\"format\":\"bob-shadow-contract-1\",\"lr_den\":16,\"lr_num\":1,\"optimizer\":\"sgd\",\"order_path\":\"/Volumes/F32WORK/training-scratch/bob/mvc/.scratch/mvc-registration-a18/init.bootstrap/order.u32.le.bin\",\"order_sha256\":\"a780f6e86eb15514ad25f04eae0d44d0f6f628c2555312369d69d776879672e7\",\"schedule_total\":3128,\"seed\":2827,\"shadow_sha256\":\"0a318217280f44468e43e891aa3cd35eb8f97a53b2f262cb0be776d08ff64c0e\",\"steps\":450,\"warmup\":200}",
  "tensors": [
    "h.0.attn.proj.bias",
    "h.0.attn.proj.weight",
    "h.0.attn.qkv.bias",
    "h.0.attn.qkv.weight",
    "h.0.ln1.bias",
    "h.0.ln1.weight",
    "h.0.ln2.bias",
    "h.0.ln2.weight",
    "h.0.mlp.fc.bias",
    "h.0.mlp.fc.weight",
    "h.0.mlp.proj.bias",
    "h.0.mlp.proj.weight",
    "h.1.attn.proj.bias",
    "h.1.attn.proj.weight",
    "h.1.attn.qkv.bias",
    "h.1.attn.qkv.weight",
    "h.1.ln1.bias",
    "h.1.ln1.weight",
    "h.1.ln2.bias",
    "h.1.ln2.weight",
    "h.1.mlp.fc.bias",
    "h.1.mlp.fc.weight",
    "h.1.mlp.proj.bias",
    "h.1.mlp.proj.weight",
    "h.2.attn.proj.bias",
    "h.2.attn.proj.weight",
    "h.2.attn.qkv.bias",
    "h.2.attn.qkv.weight",
    "h.2.ln1.bias",
    "h.2.ln1.weight",
    "h.2.ln2.bias",
    "h.2.ln2.weight",
    "h.2.mlp.fc.bias",
    "h.2.mlp.fc.weight",
    "h.2.mlp.proj.bias",
    "h.2.mlp.proj.weight",
    "h.3.attn.proj.bias",
    "h.3.attn.proj.weight",
    "h.3.attn.qkv.bias",
    "h.3.attn.qkv.weight",
    "h.3.ln1.bias",
    "h.3.ln1.weight",
    "h.3.ln2.bias",
    "h.3.ln2.weight",
    "h.3.mlp.fc.bias",
    "h.3.mlp.fc.weight",
    "h.3.mlp.proj.bias",
    "h.3.mlp.proj.weight",
    "h.4.attn.proj.bias",
    "h.4.attn.proj.weight",
    "h.4.attn.qkv.bias",
    "h.4.attn.qkv.weight",
    "h.4.ln1.bias",
    "h.4.ln1.weight",
    "h.4.ln2.bias",
    "h.4.ln2.weight",
    "h.4.mlp.fc.bias",
    "h.4.mlp.fc.weight",
    "h.4.mlp.proj.bias",
    "h.4.mlp.proj.weight",
    "ln_f.bias",
    "ln_f.weight",
    "wpe",
    "wte"
  ],
  "weight": [
    {
      "bits": 23,
      "exp": -25
    },
    {
      "bits": 23,
      "exp": -25
    },
    {
      "bits": 23,
      "exp": -27
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -28
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -31
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -29
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -26
    },
    {
      "bits": 23,
      "exp": -25
    },
    {
      "bits": 23,
      "exp": -26
    },
    {
      "bits": 23,
      "exp": -25
    },
    {
      "bits": 23,
      "exp": -29
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -31
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -31
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -30
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -26
    },
    {
      "bits": 23,
      "exp": -25
    },
    {
      "bits": 23,
      "exp": -26
    },
    {
      "bits": 23,
      "exp": -24
    },
    {
      "bits": 23,
      "exp": -30
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -31
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -31
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -30
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -26
    },
    {
      "bits": 23,
      "exp": -25
    },
    {
      "bits": 23,
      "exp": -26
    },
    {
      "bits": 23,
      "exp": -25
    },
    {
      "bits": 23,
      "exp": -30
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -31
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -31
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -30
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -26
    },
    {
      "bits": 23,
      "exp": -25
    },
    {
      "bits": 23,
      "exp": -26
    },
    {
      "bits": 23,
      "exp": -25
    },
    {
      "bits": 23,
      "exp": -29
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -31
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -30
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -30
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -26
    },
    {
      "bits": 23,
      "exp": -25
    },
    {
      "bits": 23,
      "exp": -24
    },
    {
      "bits": 23,
      "exp": -19
    },
    {
      "bits": 23,
      "exp": -23
    },
    {
      "bits": 23,
      "exp": -22
    }
  ]
}