Hugging Face's logo

mpnikhil
/
dev-0.4b

Text Classification
Transformers
Safetensors
English
modernbert
fill-mask
decision-model
routing
classification
verification
brier-score
calibration
bidirectional
Eval Results (legacy)
Model card Files
xet
 Community
dev-0.4b
File size: 2,200 Bytes
7692b80
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
{
  "train": "data/train_refine.jsonl",
  "validation": "data/validation_refine.jsonl",
  "model": "answerdotai/ModernBERT-large",
  "revision": null,
  "output": "runs/jev-unified-large",
  "device": "mps",
  "epochs": 3,
  "batch_size": 8,
  "accumulation": 4,
  "max_length": 1024,
  "max_steps": 0,
  "encoder_lr": 5e-06,
  "head_lr": 0.0001,
  "freeze_encoder": false,
  "gradient_checkpointing": true,
  "lora_rank": 0,
  "resume": "runs/jev-unified-large/trainer.pt",
  "init_checkpoint": null,
  "seed": 17,
  "schema_version": 2,
  "device_used": "mps",
  "parameters": 396880897,
  "trainable_parameters": 396880897,
  "status": "calibrated",
  "temperatures": {
    "noul": 1.3575,
    "choice": 4.2542,
    "score": 3.7097
  },
  "calibration_method": "temperature_scaling_guo_2017",
  "calibration_metrics": {
    "noul": {
      "temperature": 1.3575,
      "ece_equal_mass": 0.016,
      "accuracy": 0.9665,
      "note": "Fitted on held-out validation; on external BoolQ (n=500), reduces ECE from 0.1033 to 0.0771 (-25%) with 100% accuracy invariance."
    },
    "choice": {
      "temperature": 4.2542,
      "ece_before": 0.1887,
      "ece_after": 0.083,
      "nll_before": 2.2566,
      "nll_after": 0.7259,
      "accuracy": 0.7819
    },
    "score": {
      "temperature": 3.7097,
      "ece_before": 0.3315,
      "ece_after": 0.1165,
      "nll_before": 2.5899,
      "nll_after": 1.1448,
      "mae_before": 0.7243,
      "mae_after": 0.7165,
      "accuracy": 0.5448
    }
  },
  "before": {
    "choice_accuracy": 0.8006968641114982,
    "choice_nll": Infinity,
    "choice_recall_at_3": 0.9665505226480836,
    "score_accuracy": 0.5380281690140845,
    "score_nll": Infinity,
    "score_mae": 0.7156380733637743,
    "noul_brier": 0.05362682237181553,
    "noul_accuracy": 0.9386046511627907
  },
  "steps": 1566,
  "after": {
    "choice_accuracy": 0.7804878048780488,
    "choice_nll": Infinity,
    "choice_recall_at_3": 0.9616724738675958,
    "score_accuracy": 0.5442253521126761,
    "score_nll": Infinity,
    "score_mae": 0.7245084745783201,
    "noul_brier": 0.02923501114512599,
    "noul_accuracy": 0.9665116279069768
  },
  "elapsed_s": 10037.545325374987
}