Upload best final ModernBERT sentence scorer model

config.json

@@ -0,0 +1,85 @@
+{
+  "architectures": [
+    "ModernBertForSequenceClassification"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "classifier_activation": "gelu",
+  "classifier_bias": false,
+  "classifier_dropout": 0.0,
+  "classifier_pooling": "mean",
+  "cls_token_id": 50281,
+  "decoder_bias": true,
+  "deterministic_flash_attn": false,
+  "dtype": "float32",
+  "embedding_dropout": 0.0,
+  "eos_token_id": null,
+  "global_attn_every_n_layers": 3,
+  "gradient_checkpointing": false,
+  "hidden_activation": "gelu",
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_cutoff_factor": 2.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 1152,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-05,
+  "layer_types": [
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "local_attention": 128,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mlp_dropout": 0.0,
+  "model_type": "modernbert",
+  "norm_bias": false,
+  "norm_eps": 1e-05,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 22,
+  "pad_token_id": 50283,
+  "position_embedding_type": "absolute",
+  "problem_type": "regression",
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 160000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sep_token_id": 50282,
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.2",
+  "use_cache": false,
+  "vocab_size": 50368
+}

eval_metrics.json

@@ -0,0 +1,14 @@
+{
+  "eval_loss": 0.02754737250506878,
+  "eval_rmse": 0.16597401714640134,
+  "eval_mae": 0.12368558347225189,
+  "eval_pearson": 0.8236426997589091,
+  "eval_spearman": 0.8201133644491532,
+  "eval_article_spearman": 0.7319277523759264,
+  "eval_article_ndcg_at_3": 0.9055421605026681,
+  "eval_article_ndcg_at_5": 0.8978525730107749,
+  "eval_runtime": 80.6871,
+  "eval_samples_per_second": 84.189,
+  "eval_steps_per_second": 42.101,
+  "epoch": 3.0
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b21e00fda4edcb892168bff241428d136dbb07d5f55ae33c4362132355a72a31
+size 598436708

run_summary.md

@@ -0,0 +1,44 @@
+# CNN/DailyMail ModernBERT Pointwise Run
+
+- model_size: `base`
+- model_name: `answerdotai/ModernBERT-base`
+- resolved_model_name: `/lustre/fsmisc/dataset/HuggingFace_Models/answerdotai/ModernBERT-base`
+- target_mode: `model_balanced`
+- target_field: `salience_score_model_balanced`
+- input_field: `model_input_text`
+- train_file: `data/CNN_DM/representative_subset/salience_generation/recovery_f2_all6/sentence_scorer_data/modernbert_pointwise/train.jsonl`
+- dev_file: `data/CNN_DM/representative_subset/salience_generation/recovery_f2_all6/sentence_scorer_data/modernbert_pointwise/dev.jsonl`
+- train_rows: `59643`
+- dev_rows: `6793`
+- train_articles: `1000`
+- dev_articles: `200`
+- max_length: `4096`
+- learning_rate: `2e-05`
+- weight_decay: `0.01`
+- num_train_epochs: `3.0`
+- warmup_ratio: `0.06`
+- per_device_train_batch_size: `2`
+- per_device_eval_batch_size: `2`
+- gradient_accumulation_steps: `8`
+- effective_train_batch_size: `16`
+- eval_strategy: `epoch`
+- save_strategy: `epoch`
+- early_stopping_patience: `2`
+- bf16: `True`
+- fp16: `False`
+- gradient_checkpointing: `True`
+
+## Eval Metrics
+
+- epoch: `3.0`
+- eval_article_ndcg_at_3: `0.9055421605026681`
+- eval_article_ndcg_at_5: `0.8978525730107749`
+- eval_article_spearman: `0.7319277523759264`
+- eval_loss: `0.02754737250506878`
+- eval_mae: `0.12368558347225189`
+- eval_pearson: `0.8236426997589091`
+- eval_rmse: `0.16597401714640134`
+- eval_runtime: `80.6871`
+- eval_samples_per_second: `84.189`
+- eval_spearman: `0.8201133644491532`
+- eval_steps_per_second: `42.101`

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "is_local": true,
+  "local_files_only": true,
+  "mask_token": "[MASK]",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "[UNK]"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:059dd50704b489c05922815531df74261005f542bc83e4d7d9176c65ee518fc9
+size 5393