Upload numbers_for_paper evaluation artifacts

.gitattributes

@@ -35,3 +35,28 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 predictions/test_predictions.jsonl filter=lfs diff=lfs merge=lfs -text
 predictions/validation_predictions.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/best/test/confusion_matrix.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/best/test/confusion_matrix_normalized_pred.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/best/test/confusion_matrix_normalized_true.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/best/test/low_confidence_correct.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/best/validation/confusion_matrix.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/best/validation/confusion_matrix_normalized_pred.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/best/validation/confusion_matrix_normalized_true.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/best/validation/low_confidence_correct.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/latest/test/confusion_matrix.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/latest/test/confusion_matrix_normalized_pred.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/latest/test/confusion_matrix_normalized_true.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/latest/test/low_confidence_correct.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/latest/validation/confusion_matrix.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/latest/validation/confusion_matrix_normalized_pred.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/latest/validation/confusion_matrix_normalized_true.png filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/metrics/latest/validation/low_confidence_correct.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/predictions/best/test_predictions.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/predictions/best/validation_predictions.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/predictions/latest/test_predictions.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/predictions/latest/validation_predictions.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/source_snapshots/dataset/artifacts/split_assignments.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/source_snapshots/dataset/type_predictor_data.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/source_snapshots/dataset/type_predictor_train.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/source_snapshots/model/predictions/test_predictions.jsonl filter=lfs diff=lfs merge=lfs -text
+numbers_for_paper/source_snapshots/model/predictions/validation_predictions.jsonl filter=lfs diff=lfs merge=lfs -text

numbers_for_paper/README.md

@@ -0,0 +1,9 @@
+# TypePredictor numbers_for_paper
+
+Deterministic inference evaluation artifacts for:
+
+- Model: `U4RASD/TypePredictor` at `56b9a60244634f073635191fe0ae1bee8a5f8c62`
+- Dataset: `U4RASD/TypePrediction` at `2cc13badf71d134057168b7e96cf78d1fdfdc4cc`
+- Paper checkpoint: released selected/best root checkpoint, step 25,000.
+
+The release intentionally excludes credentials, HF caches, Python caches, and redundant model-weight files.

numbers_for_paper/data_audit/schema.json

@@ -0,0 +1,124 @@
+{
+  "fields": {
+    "end_char": {
+      "python_types_observed_sample": [
+        "int"
+      ],
+      "example": 81
+    },
+    "end_token": {
+      "python_types_observed_sample": [
+        "int"
+      ],
+      "example": 14
+    },
+    "entity": {
+      "python_types_observed_sample": [
+        "str"
+      ],
+      "example": "محافظة الناصرة"
+    },
+    "evaluation_category": {
+      "python_types_observed_sample": [
+        "str"
+      ],
+      "example": "seen_sentence_new_entity"
+    },
+    "id": {
+      "python_types_observed_sample": [
+        "str"
+      ],
+      "example": "tp_078298"
+    },
+    "raw_types": {
+      "python_types_observed_sample": [
+        "list"
+      ],
+      "example": [
+        "GPE",
+        "STATE-OR-PROVINCE"
+      ]
+    },
+    "sentence": {
+      "python_types_observed_sample": [
+        "str"
+      ],
+      "example": "سند خاقاني : وثائق طابو من الفترة العثمانية عن اراضي قرية صفورية ، محافظة الناصرة ."
+    },
+    "sources": {
+      "python_types_observed_sample": [
+        "list"
+      ],
+      "example": [
+        {
+          "dataset": "Wojood1_1_flat",
+          "split": "train"
+        },
+        {
+          "dataset": "Wojood1_1_nested",
+          "split": "train"
+        },
+        {
+          "dataset": "WojoodFine",
+          "split": "train"
+        },
+        {
+          "dataset": "WojoodFine-Flat",
+          "split": "split70"
+        }
+      ]
+    },
+    "start_char": {
+      "python_types_observed_sample": [
+        "int"
+      ],
+      "example": 67
+    },
+    "start_token": {
+      "python_types_observed_sample": [
+        "int"
+      ],
+      "example": 13
+    },
+    "type": {
+      "python_types_observed_sample": [
+        "str"
+      ],
+      "example": "GPE"
+    }
+  },
+  "splits": {
+    "validation": {
+      "rows": 12600,
+      "fields": [
+        "end_char",
+        "end_token",
+        "entity",
+        "evaluation_category",
+        "id",
+        "raw_types",
+        "sentence",
+        "sources",
+        "start_char",
+        "start_token",
+        "type"
+      ]
+    },
+    "test": {
+      "rows": 12600,
+      "fields": [
+        "end_char",
+        "end_token",
+        "entity",
+        "evaluation_category",
+        "id",
+        "raw_types",
+        "sentence",
+        "sources",
+        "start_char",
+        "start_token",
+        "type"
+      ]
+    }
+  }
+}

numbers_for_paper/data_audit/test_audit.json

@@ -0,0 +1,95 @@
+{
+  "split": "test",
+  "row_count": 12600,
+  "expected_row_count": 12600,
+  "row_count_ok": true,
+  "unique_ids": 12600,
+  "duplicate_ids": [],
+  "fields": [
+    "end_char",
+    "end_token",
+    "entity",
+    "evaluation_category",
+    "id",
+    "raw_types",
+    "sentence",
+    "sources",
+    "start_char",
+    "start_token",
+    "type"
+  ],
+  "identified_fields": {
+    "sentence": "sentence",
+    "mention": "entity",
+    "start": "start_char",
+    "end": "end_char",
+    "gold_type": "type",
+    "evaluation_category": "evaluation_category"
+  },
+  "null_counts": {},
+  "span_mismatch_count": 0,
+  "span_mismatch_examples": [],
+  "token_span_mismatch_count": 0,
+  "token_span_mismatch_examples": [],
+  "unknown_label_count": 0,
+  "unknown_label_examples": [],
+  "duplicate_mention_key_group_count": 0,
+  "conflicting_span_type_group_count": 0,
+  "conflicting_span_type_examples": [],
+  "label_counts": {
+    "GPE": 2691,
+    "ORG": 2659,
+    "DATE": 1977,
+    "PERS": 1071,
+    "NORP": 1021,
+    "ORDINAL": 781,
+    "OCC": 770,
+    "EVENT": 377,
+    "CARDINAL": 372,
+    "LOC": 239,
+    "WEBSITE": 148,
+    "FAC": 129,
+    "LAW": 91,
+    "TIME": 87,
+    "MONEY": 42,
+    "CURR": 41,
+    "LANGUAGE": 33,
+    "PERCENT": 31,
+    "PRODUCT": 19,
+    "QUANTITY": 10,
+    "UNIT": 11
+  },
+  "category_counts": {
+    "seen_sentence_new_entity": 11606,
+    "unseen_sentence": 994
+  },
+  "unique_sentence_count": 10161,
+  "mention_length_summary_chars": {
+    "count": 12600,
+    "min": 1.0,
+    "mean": 12.05984126984127,
+    "median": 10.0,
+    "p95": 29.0,
+    "p99": 40.01000000000022,
+    "max": 159.0
+  },
+  "sentence_length_summary_chars": {
+    "count": 12600,
+    "min": 4.0,
+    "mean": 215.19730158730158,
+    "median": 108.0,
+    "p95": 727.0,
+    "p99": 1716.4400000000096,
+    "max": 6613.0
+  },
+  "entity_occurrence_count_summary": {
+    "count": 12600,
+    "min": 1.0,
+    "mean": 1.1625396825396825,
+    "median": 1.0,
+    "p95": 2.0,
+    "p99": 4.0,
+    "max": 17.0
+  },
+  "rows_not_in_documented_categories": 0
+}

numbers_for_paper/data_audit/test_label_distribution.csv

@@ -0,0 +1,22 @@
+label,count
+GPE,2691
+ORG,2659
+DATE,1977
+PERS,1071
+NORP,1021
+ORDINAL,781
+OCC,770
+EVENT,377
+CARDINAL,372
+LOC,239
+WEBSITE,148
+FAC,129
+LAW,91
+TIME,87
+MONEY,42
+CURR,41
+LANGUAGE,33
+PERCENT,31
+PRODUCT,19
+QUANTITY,10
+UNIT,11

numbers_for_paper/data_audit/validation_audit.json

@@ -0,0 +1,95 @@
+{
+  "split": "validation",
+  "row_count": 12600,
+  "expected_row_count": 12600,
+  "row_count_ok": true,
+  "unique_ids": 12600,
+  "duplicate_ids": [],
+  "fields": [
+    "end_char",
+    "end_token",
+    "entity",
+    "evaluation_category",
+    "id",
+    "raw_types",
+    "sentence",
+    "sources",
+    "start_char",
+    "start_token",
+    "type"
+  ],
+  "identified_fields": {
+    "sentence": "sentence",
+    "mention": "entity",
+    "start": "start_char",
+    "end": "end_char",
+    "gold_type": "type",
+    "evaluation_category": "evaluation_category"
+  },
+  "null_counts": {},
+  "span_mismatch_count": 0,
+  "span_mismatch_examples": [],
+  "token_span_mismatch_count": 0,
+  "token_span_mismatch_examples": [],
+  "unknown_label_count": 0,
+  "unknown_label_examples": [],
+  "duplicate_mention_key_group_count": 0,
+  "conflicting_span_type_group_count": 0,
+  "conflicting_span_type_examples": [],
+  "label_counts": {
+    "GPE": 2690,
+    "ORG": 2660,
+    "DATE": 1976,
+    "PERS": 1072,
+    "NORP": 1022,
+    "ORDINAL": 780,
+    "OCC": 769,
+    "EVENT": 377,
+    "CARDINAL": 373,
+    "LOC": 239,
+    "WEBSITE": 148,
+    "FAC": 130,
+    "LAW": 90,
+    "TIME": 87,
+    "MONEY": 42,
+    "CURR": 41,
+    "LANGUAGE": 33,
+    "PERCENT": 31,
+    "PRODUCT": 19,
+    "QUANTITY": 11,
+    "UNIT": 10
+  },
+  "category_counts": {
+    "seen_sentence_new_entity": 11661,
+    "unseen_sentence": 939
+  },
+  "unique_sentence_count": 10139,
+  "mention_length_summary_chars": {
+    "count": 12600,
+    "min": 1.0,
+    "mean": 11.997460317460318,
+    "median": 11.0,
+    "p95": 29.0,
+    "p99": 40.0,
+    "max": 105.0
+  },
+  "sentence_length_summary_chars": {
+    "count": 12600,
+    "min": 3.0,
+    "mean": 208.72746031746033,
+    "median": 109.0,
+    "p95": 694.0,
+    "p99": 1708.0,
+    "max": 6613.0
+  },
+  "entity_occurrence_count_summary": {
+    "count": 12600,
+    "min": 1.0,
+    "mean": 1.1506349206349207,
+    "median": 1.0,
+    "p95": 2.0,
+    "p99": 4.0,
+    "max": 17.0
+  },
+  "rows_not_in_documented_categories": 0
+}

numbers_for_paper/data_audit/validation_label_distribution.csv

@@ -0,0 +1,22 @@
+label,count
+GPE,2690
+ORG,2660
+DATE,1976
+PERS,1072
+NORP,1022
+ORDINAL,780
+OCC,769
+EVENT,377
+CARDINAL,373
+LOC,239
+WEBSITE,148
+FAC,130
+LAW,90
+TIME,87
+MONEY,42
+CURR,41
+LANGUAGE,33
+PERCENT,31
+PRODUCT,19
+QUANTITY,11
+UNIT,10

numbers_for_paper/metrics/best/test/category_metrics.csv

@@ -0,0 +1,5 @@
+row_count,correct_count,accuracy,micro_precision,micro_recall,micro_f1,macro_all_labels_precision,macro_all_labels_recall,macro_all_labels_f1,weighted_precision,weighted_recall,weighted_f1,macro_present_labels_precision,macro_present_labels_recall,macro_present_labels_f1,macro_precision_all_labels,macro_recall_all_labels,macro_f1_all_labels,macro_precision_present_labels,macro_recall_present_labels,macro_f1_present_labels,top3_accuracy,top5_accuracy,log_loss,mean_top1_confidence,median_top1_confidence,p05_top1_confidence,p95_top1_confidence,mean_confidence_correct,mean_confidence_incorrect,category_field,category_value
+11606,11383,0.9807858004480441,0.9807858004480441,0.9807858004480441,0.9807858004480441,0.970282742754196,0.9589517514121707,0.9641131075953279,0.9807493542827415,0.9807858004480441,0.9807123412635281,0.970282742754196,0.9589517514121707,0.9641131075953279,0.970282742754196,0.9589517514121707,0.9641131075953279,0.970282742754196,0.9589517514121707,0.9641131075953279,0.9944856108909185,0.9967258314664829,0.1489705142491425,0.9957778798790198,1.0,0.9996167719364166,1.0,0.9980196648408332,0.8813463183529174,evaluation_category,seen_sentence_new_entity
+994,957,0.9627766599597586,0.9627766599597586,0.9627766599597586,0.9627766599597586,0.8621871812676783,0.8440708315067772,0.8506415101139764,0.9615439764829642,0.9627766599597586,0.9617803672434098,0.9052965403310622,0.886274373082116,0.8931735856196752,0.8621871812676783,0.8440708315067772,0.8506415101139764,0.9052965403310622,0.886274373082116,0.8931735856196752,0.9818913480885312,0.9899396378269618,0.24372136249773205,0.9908601785869425,0.9999997615814209,0.9958629250526428,1.0,0.9978639928773901,0.8097074684259054,evaluation_category,unseen_sentence
+12413,12169,0.9803431885926045,0.9803431885926045,0.9803431885926045,0.9803431885926045,0.649632511071306,0.6428307010783108,0.6461364714198562,0.9806357996885862,0.9803431885926045,0.9804517199674248,0.9744487666069592,0.9642460516174661,0.9692047071297843,0.649632511071306,0.6428307010783108,0.6461364714198562,0.9744487666069592,0.9642460516174661,0.9692047071297843,0.993796825908322,0.996294207685491,0.15243328859391145,0.9957548734558432,1.0,0.9996043562889099,1.0,0.9981771630414638,0.874948144081186,label_frequency_category,frequent_label_gold_support_ge_500_dataset
+187,171,0.9144385026737968,0.9144385026737968,0.9144385026737968,0.9144385026737968,0.3248890757143095,0.310637292786805,0.31710717645832703,0.977959909496639,0.9144385026737968,0.9440013868098467,0.9746672271429286,0.9319118783604151,0.9513215293749812,0.3248890757143095,0.310637292786805,0.31710717645832703,0.9746672271429286,0.9319118783604151,0.9513215293749812,0.9732620320855615,0.9893048128342246,0.4227615581928923,0.9711649582029026,0.9999908208847046,0.7717686951160432,0.9999997615814209,0.985940286639141,0.8132536355406046,label_frequency_category,rare_label_gold_support_lt_500_dataset

numbers_for_paper/metrics/best/test/category_metrics.json

@@ -0,0 +1,142 @@
+{
+  "evaluation_category": {
+    "seen_sentence_new_entity": {
+      "row_count": 11606,
+      "correct_count": 11383,
+      "accuracy": 0.9807858004480441,
+      "micro_precision": 0.9807858004480441,
+      "micro_recall": 0.9807858004480441,
+      "micro_f1": 0.9807858004480441,
+      "macro_all_labels_precision": 0.970282742754196,
+      "macro_all_labels_recall": 0.9589517514121707,
+      "macro_all_labels_f1": 0.9641131075953279,
+      "weighted_precision": 0.9807493542827415,
+      "weighted_recall": 0.9807858004480441,
+      "weighted_f1": 0.9807123412635281,
+      "macro_present_labels_precision": 0.970282742754196,
+      "macro_present_labels_recall": 0.9589517514121707,
+      "macro_present_labels_f1": 0.9641131075953279,
+      "macro_precision_all_labels": 0.970282742754196,
+      "macro_recall_all_labels": 0.9589517514121707,
+      "macro_f1_all_labels": 0.9641131075953279,
+      "macro_precision_present_labels": 0.970282742754196,
+      "macro_recall_present_labels": 0.9589517514121707,
+      "macro_f1_present_labels": 0.9641131075953279,
+      "top3_accuracy": 0.9944856108909185,
+      "top5_accuracy": 0.9967258314664829,
+      "log_loss": 0.1489705142491425,
+      "mean_top1_confidence": 0.9957778798790198,
+      "median_top1_confidence": 1.0,
+      "p05_top1_confidence": 0.9996167719364166,
+      "p95_top1_confidence": 1.0,
+      "mean_confidence_correct": 0.9980196648408332,
+      "mean_confidence_incorrect": 0.8813463183529174,
+      "category_field": "evaluation_category",
+      "category_value": "seen_sentence_new_entity"
+    },
+    "unseen_sentence": {
+      "row_count": 994,
+      "correct_count": 957,
+      "accuracy": 0.9627766599597586,
+      "micro_precision": 0.9627766599597586,
+      "micro_recall": 0.9627766599597586,
+      "micro_f1": 0.9627766599597586,
+      "macro_all_labels_precision": 0.8621871812676783,
+      "macro_all_labels_recall": 0.8440708315067772,
+      "macro_all_labels_f1": 0.8506415101139764,
+      "weighted_precision": 0.9615439764829642,
+      "weighted_recall": 0.9627766599597586,
+      "weighted_f1": 0.9617803672434098,
+      "macro_present_labels_precision": 0.9052965403310622,
+      "macro_present_labels_recall": 0.886274373082116,
+      "macro_present_labels_f1": 0.8931735856196752,
+      "macro_precision_all_labels": 0.8621871812676783,
+      "macro_recall_all_labels": 0.8440708315067772,
+      "macro_f1_all_labels": 0.8506415101139764,
+      "macro_precision_present_labels": 0.9052965403310622,
+      "macro_recall_present_labels": 0.886274373082116,
+      "macro_f1_present_labels": 0.8931735856196752,
+      "top3_accuracy": 0.9818913480885312,
+      "top5_accuracy": 0.9899396378269618,
+      "log_loss": 0.24372136249773205,
+      "mean_top1_confidence": 0.9908601785869425,
+      "median_top1_confidence": 0.9999997615814209,
+      "p05_top1_confidence": 0.9958629250526428,
+      "p95_top1_confidence": 1.0,
+      "mean_confidence_correct": 0.9978639928773901,
+      "mean_confidence_incorrect": 0.8097074684259054,
+      "category_field": "evaluation_category",
+      "category_value": "unseen_sentence"
+    }
+  },
+  "label_frequency_category": {
+    "frequent_label_gold_support_ge_500_dataset": {
+      "row_count": 12413,
+      "correct_count": 12169,
+      "accuracy": 0.9803431885926045,
+      "micro_precision": 0.9803431885926045,
+      "micro_recall": 0.9803431885926045,
+      "micro_f1": 0.9803431885926045,
+      "macro_all_labels_precision": 0.649632511071306,
+      "macro_all_labels_recall": 0.6428307010783108,
+      "macro_all_labels_f1": 0.6461364714198562,
+      "weighted_precision": 0.9806357996885862,
+      "weighted_recall": 0.9803431885926045,
+      "weighted_f1": 0.9804517199674248,
+      "macro_present_labels_precision": 0.9744487666069592,
+      "macro_present_labels_recall": 0.9642460516174661,
+      "macro_present_labels_f1": 0.9692047071297843,
+      "macro_precision_all_labels": 0.649632511071306,
+      "macro_recall_all_labels": 0.6428307010783108,
+      "macro_f1_all_labels": 0.6461364714198562,
+      "macro_precision_present_labels": 0.9744487666069592,
+      "macro_recall_present_labels": 0.9642460516174661,
+      "macro_f1_present_labels": 0.9692047071297843,
+      "top3_accuracy": 0.993796825908322,
+      "top5_accuracy": 0.996294207685491,
+      "log_loss": 0.15243328859391145,
+      "mean_top1_confidence": 0.9957548734558432,
+      "median_top1_confidence": 1.0,
+      "p05_top1_confidence": 0.9996043562889099,
+      "p95_top1_confidence": 1.0,
+      "mean_confidence_correct": 0.9981771630414638,
+      "mean_confidence_incorrect": 0.874948144081186,
+      "category_field": "label_frequency_category",
+      "category_value": "frequent_label_gold_support_ge_500_dataset"
+    },
+    "rare_label_gold_support_lt_500_dataset": {
+      "row_count": 187,
+      "correct_count": 171,
+      "accuracy": 0.9144385026737968,
+      "micro_precision": 0.9144385026737968,
+      "micro_recall": 0.9144385026737968,
+      "micro_f1": 0.9144385026737968,
+      "macro_all_labels_precision": 0.3248890757143095,
+      "macro_all_labels_recall": 0.310637292786805,
+      "macro_all_labels_f1": 0.31710717645832703,
+      "weighted_precision": 0.977959909496639,
+      "weighted_recall": 0.9144385026737968,
+      "weighted_f1": 0.9440013868098467,
+      "macro_present_labels_precision": 0.9746672271429286,
+      "macro_present_labels_recall": 0.9319118783604151,
+      "macro_present_labels_f1": 0.9513215293749812,
+      "macro_precision_all_labels": 0.3248890757143095,
+      "macro_recall_all_labels": 0.310637292786805,
+      "macro_f1_all_labels": 0.31710717645832703,
+      "macro_precision_present_labels": 0.9746672271429286,
+      "macro_recall_present_labels": 0.9319118783604151,
+      "macro_f1_present_labels": 0.9513215293749812,
+      "top3_accuracy": 0.9732620320855615,
+      "top5_accuracy": 0.9893048128342246,
+      "log_loss": 0.4227615581928923,
+      "mean_top1_confidence": 0.9711649582029026,
+      "median_top1_confidence": 0.9999908208847046,
+      "p05_top1_confidence": 0.7717686951160432,
+      "p95_top1_confidence": 0.9999997615814209,
+      "mean_confidence_correct": 0.985940286639141,
+      "mean_confidence_incorrect": 0.8132536355406046,
+      "category_field": "label_frequency_category",
+      "category_value": "rare_label_gold_support_lt_500_dataset"
+    }
+  }
+}

numbers_for_paper/metrics/best/test/class_wise_most_common_confusions.csv

@@ -0,0 +1,74 @@
+gold,predicted,count
+GPE,ORG,21
+GPE,NORP,6
+GPE,DATE,4
+GPE,LOC,3
+GPE,PERS,2
+ORG,GPE,36
+ORG,NORP,6
+ORG,PERS,5
+ORG,EVENT,3
+ORG,FAC,3
+DATE,GPE,2
+DATE,PERS,2
+DATE,TIME,2
+DATE,NORP,1
+DATE,LOC,1
+PERS,GPE,4
+PERS,ORG,4
+PERS,OCC,4
+PERS,DATE,2
+PERS,NORP,2
+NORP,ORG,7
+NORP,PERS,5
+NORP,GPE,3
+NORP,DATE,1
+NORP,OCC,1
+ORDINAL,ORG,5
+ORDINAL,DATE,3
+ORDINAL,CARDINAL,2
+ORDINAL,PERS,1
+ORDINAL,NORP,1
+OCC,ORG,3
+OCC,GPE,1
+OCC,PERS,1
+OCC,NORP,1
+OCC,WEBSITE,1
+EVENT,ORG,8
+EVENT,NORP,2
+EVENT,FAC,2
+EVENT,GPE,1
+EVENT,PERS,1
+CARDINAL,ORDINAL,8
+CARDINAL,DATE,4
+CARDINAL,OCC,3
+CARDINAL,GPE,1
+CARDINAL,PERS,1
+LOC,GPE,11
+LOC,ORG,2
+LOC,NORP,2
+LOC,PERS,1
+WEBSITE,ORG,1
+WEBSITE,TIME,1
+FAC,GPE,9
+FAC,PERS,2
+FAC,DATE,1
+FAC,LOC,1
+TIME,ORDINAL,4
+TIME,GPE,2
+TIME,FAC,2
+TIME,LANGUAGE,2
+TIME,DATE,1
+MONEY,DATE,1
+MONEY,CARDINAL,1
+MONEY,CURR,1
+MONEY,PERCENT,1
+MONEY,QUANTITY,1
+CURR,DATE,1
+LANGUAGE,NORP,2
+LANGUAGE,GPE,1
+LANGUAGE,PERS,1
+LANGUAGE,TIME,1
+PERCENT,CARDINAL,3
+PERCENT,MONEY,1
+PRODUCT,ORDINAL,1

numbers_for_paper/metrics/best/test/classification_report.csv

@@ -0,0 +1,25 @@
+,precision,recall,f1-score,support
+GPE,0.9738970588235294,0.9843924191750278,0.9791166143041952,2691.0
+ORG,0.9807692307692307,0.9781872884543061,0.9794765580869893,2659.0
+DATE,0.9904378459989934,0.9954476479514416,0.992936427850656,1977.0
+PERS,0.9795348837209302,0.9831932773109243,0.9813606710158435,1071.0
+NORP,0.9765395894428153,0.9784524975514202,0.9774951076320939,1021.0
+ORDINAL,0.9820971867007673,0.9833546734955185,0.982725527831094,781.0
+OCC,0.9870466321243523,0.9896103896103896,0.9883268482490273,770.0
+EVENT,0.9864130434782609,0.9628647214854111,0.974496644295302,377.0
+CARDINAL,0.9779005524861878,0.9516129032258065,0.9645776566757494,372.0
+LOC,0.9695652173913043,0.9330543933054394,0.9509594882729211,239.0
+WEBSITE,0.9668874172185431,0.9864864864864865,0.9765886287625418,148.0
+FAC,0.928,0.8992248062015504,0.9133858267716536,129.0
+LAW,1.0,1.0,1.0,91.0
+TIME,0.9156626506024096,0.8735632183908046,0.8941176470588236,87.0
+MONEY,0.9736842105263158,0.8809523809523809,0.925,42.0
+CURR,0.975609756097561,0.975609756097561,0.975609756097561,41.0
+LANGUAGE,0.875,0.8484848484848485,0.8615384615384616,33.0
+PERCENT,0.9642857142857143,0.8709677419354839,0.9152542372881356,31.0
+PRODUCT,1.0,0.9473684210526315,0.972972972972973,19.0
+QUANTITY,0.9090909090909091,1.0,0.9523809523809523,10.0
+UNIT,1.0,1.0,1.0,11.0
+accuracy,0.9793650793650793,0.9793650793650793,0.9793650793650793,0.9793650793650793
+macro avg,0.9672581856551346,0.9534679938651157,0.9599200012897606,12600.0
+weighted avg,0.9793254669511197,0.9793650793650793,0.9792974524068463,12600.0

numbers_for_paper/metrics/best/test/classification_report.json

@@ -0,0 +1,141 @@
+{
+  "GPE": {
+    "precision": 0.9738970588235294,
+    "recall": 0.9843924191750278,
+    "f1-score": 0.9791166143041952,
+    "support": 2691.0
+  },
+  "ORG": {
+    "precision": 0.9807692307692307,
+    "recall": 0.9781872884543061,
+    "f1-score": 0.9794765580869893,
+    "support": 2659.0
+  },
+  "DATE": {
+    "precision": 0.9904378459989934,
+    "recall": 0.9954476479514416,
+    "f1-score": 0.992936427850656,
+    "support": 1977.0
+  },
+  "PERS": {
+    "precision": 0.9795348837209302,
+    "recall": 0.9831932773109243,
+    "f1-score": 0.9813606710158435,
+    "support": 1071.0
+  },
+  "NORP": {
+    "precision": 0.9765395894428153,
+    "recall": 0.9784524975514202,
+    "f1-score": 0.9774951076320939,
+    "support": 1021.0
+  },
+  "ORDINAL": {
+    "precision": 0.9820971867007673,
+    "recall": 0.9833546734955185,
+    "f1-score": 0.982725527831094,
+    "support": 781.0
+  },
+  "OCC": {
+    "precision": 0.9870466321243523,
+    "recall": 0.9896103896103896,
+    "f1-score": 0.9883268482490273,
+    "support": 770.0
+  },
+  "EVENT": {
+    "precision": 0.9864130434782609,
+    "recall": 0.9628647214854111,
+    "f1-score": 0.974496644295302,
+    "support": 377.0
+  },
+  "CARDINAL": {
+    "precision": 0.9779005524861878,
+    "recall": 0.9516129032258065,
+    "f1-score": 0.9645776566757494,
+    "support": 372.0
+  },
+  "LOC": {
+    "precision": 0.9695652173913043,
+    "recall": 0.9330543933054394,
+    "f1-score": 0.9509594882729211,
+    "support": 239.0
+  },
+  "WEBSITE": {
+    "precision": 0.9668874172185431,
+    "recall": 0.9864864864864865,
+    "f1-score": 0.9765886287625418,
+    "support": 148.0
+  },
+  "FAC": {
+    "precision": 0.928,
+    "recall": 0.8992248062015504,
+    "f1-score": 0.9133858267716536,
+    "support": 129.0
+  },
+  "LAW": {
+    "precision": 1.0,
+    "recall": 1.0,
+    "f1-score": 1.0,
+    "support": 91.0
+  },
+  "TIME": {
+    "precision": 0.9156626506024096,
+    "recall": 0.8735632183908046,
+    "f1-score": 0.8941176470588236,
+    "support": 87.0
+  },
+  "MONEY": {
+    "precision": 0.9736842105263158,
+    "recall": 0.8809523809523809,
+    "f1-score": 0.925,
+    "support": 42.0
+  },
+  "CURR": {
+    "precision": 0.975609756097561,
+    "recall": 0.975609756097561,
+    "f1-score": 0.975609756097561,
+    "support": 41.0
+  },
+  "LANGUAGE": {
+    "precision": 0.875,
+    "recall": 0.8484848484848485,
+    "f1-score": 0.8615384615384616,
+    "support": 33.0
+  },
+  "PERCENT": {
+    "precision": 0.9642857142857143,
+    "recall": 0.8709677419354839,
+    "f1-score": 0.9152542372881356,
+    "support": 31.0
+  },
+  "PRODUCT": {
+    "precision": 1.0,
+    "recall": 0.9473684210526315,
+    "f1-score": 0.972972972972973,
+    "support": 19.0
+  },
+  "QUANTITY": {
+    "precision": 0.9090909090909091,
+    "recall": 1.0,
+    "f1-score": 0.9523809523809523,
+    "support": 10.0
+  },
+  "UNIT": {
+    "precision": 1.0,
+    "recall": 1.0,
+    "f1-score": 1.0,
+    "support": 11.0
+  },
+  "accuracy": 0.9793650793650793,
+  "macro avg": {
+    "precision": 0.9672581856551346,
+    "recall": 0.9534679938651157,
+    "f1-score": 0.9599200012897606,
+    "support": 12600.0
+  },
+  "weighted avg": {
+    "precision": 0.9793254669511197,
+    "recall": 0.9793650793650793,
+    "f1-score": 0.9792974524068463,
+    "support": 12600.0
+  }
+}

numbers_for_paper/metrics/best/test/confusion_matrix.csv

@@ -0,0 +1,22 @@
+,GPE,ORG,DATE,PERS,NORP,ORDINAL,OCC,EVENT,CARDINAL,LOC,WEBSITE,FAC,LAW,TIME,MONEY,CURR,LANGUAGE,PERCENT,PRODUCT,QUANTITY,UNIT
+GPE,2649,21,4,2,6,0,1,1,1,3,2,1,0,0,0,0,0,0,0,0,0
+ORG,36,2601,1,5,6,0,1,3,0,2,1,3,0,0,0,0,0,0,0,0,0
+DATE,2,0,1968,2,1,0,0,0,0,1,0,0,0,2,0,0,1,0,0,0,0
+PERS,4,4,2,1053,2,1,4,0,0,0,1,0,0,0,0,0,0,0,0,0,0
+NORP,3,7,1,5,999,0,1,1,1,0,0,1,0,1,0,0,1,0,0,0,0
+ORDINAL,0,5,3,1,1,768,0,0,2,0,0,0,0,1,0,0,0,0,0,0,0
+OCC,1,3,0,1,1,0,762,0,0,0,1,0,0,1,0,0,0,0,0,0,0
+EVENT,1,8,0,1,2,0,0,363,0,0,0,2,0,0,0,0,0,0,0,0,0
+CARDINAL,1,0,4,1,1,8,3,0,354,0,0,0,0,0,0,0,0,0,0,0,0
+LOC,11,2,0,1,2,0,0,0,0,223,0,0,0,0,0,0,0,0,0,0,0
+WEBSITE,0,1,0,0,0,0,0,0,0,0,146,0,0,1,0,0,0,0,0,0,0
+FAC,9,0,1,2,0,0,0,0,0,1,0,116,0,0,0,0,0,0,0,0,0
+LAW,0,0,0,0,0,0,0,0,0,0,0,0,91,0,0,0,0,0,0,0,0
+TIME,2,0,1,0,0,4,0,0,0,0,0,2,0,76,0,0,2,0,0,0,0
+MONEY,0,0,1,0,0,0,0,0,1,0,0,0,0,0,37,1,0,1,0,1,0
+CURR,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,40,0,0,0,0,0
+LANGUAGE,1,0,0,1,2,0,0,0,0,0,0,0,0,1,0,0,28,0,0,0,0
+PERCENT,0,0,0,0,0,0,0,0,3,0,0,0,0,0,1,0,0,27,0,0,0
+PRODUCT,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,18,0,0
+QUANTITY,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,0
+UNIT,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11

numbers_for_paper/metrics/best/test/confusion_matrix_normalized_pred.csv

@@ -0,0 +1,22 @@
+,GPE,ORG,DATE,PERS,NORP,ORDINAL,OCC,EVENT,CARDINAL,LOC,WEBSITE,FAC,LAW,TIME,MONEY,CURR,LANGUAGE,PERCENT,PRODUCT,QUANTITY,UNIT
+GPE,0.9738970588235294,0.007918552036199095,0.0020130850528434826,0.0018604651162790699,0.005865102639296188,0.0,0.0012953367875647669,0.002717391304347826,0.0027624309392265192,0.013043478260869565,0.013245033112582781,0.008,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+ORG,0.013235294117647059,0.9807692307692307,0.0005032712632108706,0.004651162790697674,0.005865102639296188,0.0,0.0012953367875647669,0.008152173913043478,0.0,0.008695652173913044,0.006622516556291391,0.024,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+DATE,0.0007352941176470588,0.0,0.9904378459989934,0.0018604651162790699,0.0009775171065493646,0.0,0.0,0.0,0.0,0.004347826086956522,0.0,0.0,0.0,0.024096385542168676,0.0,0.0,0.03125,0.0,0.0,0.0,0.0
+PERS,0.0014705882352941176,0.0015082956259426848,0.0010065425264217413,0.9795348837209302,0.0019550342130987292,0.0012787723785166241,0.0051813471502590676,0.0,0.0,0.0,0.006622516556291391,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+NORP,0.0011029411764705882,0.0026395173453996985,0.0005032712632108706,0.004651162790697674,0.9765395894428153,0.0,0.0012953367875647669,0.002717391304347826,0.0027624309392265192,0.0,0.0,0.008,0.0,0.012048192771084338,0.0,0.0,0.03125,0.0,0.0,0.0,0.0
+ORDINAL,0.0,0.001885369532428356,0.001509813789632612,0.0009302325581395349,0.0009775171065493646,0.9820971867007673,0.0,0.0,0.0055248618784530384,0.0,0.0,0.0,0.0,0.012048192771084338,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+OCC,0.0003676470588235294,0.0011312217194570137,0.0,0.0009302325581395349,0.0009775171065493646,0.0,0.9870466321243523,0.0,0.0,0.0,0.006622516556291391,0.0,0.0,0.012048192771084338,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+EVENT,0.0003676470588235294,0.0030165912518853697,0.0,0.0009302325581395349,0.0019550342130987292,0.0,0.0,0.9864130434782609,0.0,0.0,0.0,0.016,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+CARDINAL,0.0003676470588235294,0.0,0.0020130850528434826,0.0009302325581395349,0.0009775171065493646,0.010230179028132993,0.0038860103626943004,0.0,0.9779005524861878,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+LOC,0.004044117647058824,0.0007541478129713424,0.0,0.0009302325581395349,0.0019550342130987292,0.0,0.0,0.0,0.0,0.9695652173913043,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+WEBSITE,0.0,0.0003770739064856712,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9668874172185431,0.0,0.0,0.012048192771084338,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+FAC,0.0033088235294117647,0.0,0.0005032712632108706,0.0018604651162790699,0.0,0.0,0.0,0.0,0.0,0.004347826086956522,0.0,0.928,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+LAW,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+TIME,0.0007352941176470588,0.0,0.0005032712632108706,0.0,0.0,0.005115089514066497,0.0,0.0,0.0,0.0,0.0,0.016,0.0,0.9156626506024096,0.0,0.0,0.0625,0.0,0.0,0.0,0.0
+MONEY,0.0,0.0,0.0005032712632108706,0.0,0.0,0.0,0.0,0.0,0.0027624309392265192,0.0,0.0,0.0,0.0,0.0,0.9736842105263158,0.024390243902439025,0.0,0.03571428571428571,0.0,0.09090909090909091,0.0
+CURR,0.0,0.0,0.0005032712632108706,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.975609756097561,0.0,0.0,0.0,0.0,0.0
+LANGUAGE,0.0003676470588235294,0.0,0.0,0.0009302325581395349,0.0019550342130987292,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.012048192771084338,0.0,0.0,0.875,0.0,0.0,0.0,0.0
+PERCENT,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008287292817679558,0.0,0.0,0.0,0.0,0.0,0.02631578947368421,0.0,0.0,0.9642857142857143,0.0,0.0,0.0
+PRODUCT,0.0,0.0,0.0,0.0,0.0,0.0012787723785166241,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0
+QUANTITY,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9090909090909091,0.0
+UNIT,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0

numbers_for_paper/metrics/best/test/confusion_matrix_normalized_true.csv

@@ -0,0 +1,22 @@
+,GPE,ORG,DATE,PERS,NORP,ORDINAL,OCC,EVENT,CARDINAL,LOC,WEBSITE,FAC,LAW,TIME,MONEY,CURR,LANGUAGE,PERCENT,PRODUCT,QUANTITY,UNIT
+GPE,0.9843924191750278,0.007803790412486065,0.0014864362690449647,0.0007432181345224824,0.002229654403567447,0.0,0.0003716090672612412,0.0003716090672612412,0.0003716090672612412,0.0011148272017837235,0.0007432181345224824,0.0003716090672612412,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+ORG,0.013538924407672057,0.9781872884543061,0.000376081233546446,0.00188040616773223,0.0022564874012786762,0.0,0.000376081233546446,0.0011282437006393381,0.0,0.000752162467092892,0.000376081233546446,0.0011282437006393381,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+DATE,0.0010116337885685382,0.0,0.9954476479514416,0.0010116337885685382,0.0005058168942842691,0.0,0.0,0.0,0.0,0.0005058168942842691,0.0,0.0,0.0,0.0010116337885685382,0.0,0.0,0.0005058168942842691,0.0,0.0,0.0,0.0
+PERS,0.003734827264239029,0.003734827264239029,0.0018674136321195146,0.9831932773109243,0.0018674136321195146,0.0009337068160597573,0.003734827264239029,0.0,0.0,0.0,0.0009337068160597573,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+NORP,0.002938295788442703,0.0068560235063663075,0.0009794319294809011,0.004897159647404506,0.9784524975514202,0.0,0.0009794319294809011,0.0009794319294809011,0.0009794319294809011,0.0,0.0,0.0009794319294809011,0.0,0.0009794319294809011,0.0,0.0,0.0009794319294809011,0.0,0.0,0.0,0.0
+ORDINAL,0.0,0.006402048655569782,0.0038412291933418692,0.0012804097311139564,0.0012804097311139564,0.9833546734955185,0.0,0.0,0.002560819462227913,0.0,0.0,0.0,0.0,0.0012804097311139564,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+OCC,0.0012987012987012987,0.003896103896103896,0.0,0.0012987012987012987,0.0012987012987012987,0.0,0.9896103896103896,0.0,0.0,0.0,0.0012987012987012987,0.0,0.0,0.0012987012987012987,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+EVENT,0.002652519893899204,0.021220159151193633,0.0,0.002652519893899204,0.005305039787798408,0.0,0.0,0.9628647214854111,0.0,0.0,0.0,0.005305039787798408,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+CARDINAL,0.002688172043010753,0.0,0.010752688172043012,0.002688172043010753,0.002688172043010753,0.021505376344086023,0.008064516129032258,0.0,0.9516129032258065,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+LOC,0.04602510460251046,0.008368200836820083,0.0,0.0041841004184100415,0.008368200836820083,0.0,0.0,0.0,0.0,0.9330543933054394,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+WEBSITE,0.0,0.006756756756756757,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9864864864864865,0.0,0.0,0.006756756756756757,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+FAC,0.06976744186046512,0.0,0.007751937984496124,0.015503875968992248,0.0,0.0,0.0,0.0,0.0,0.007751937984496124,0.0,0.8992248062015504,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+LAW,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+TIME,0.022988505747126436,0.0,0.011494252873563218,0.0,0.0,0.04597701149425287,0.0,0.0,0.0,0.0,0.0,0.022988505747126436,0.0,0.8735632183908046,0.0,0.0,0.022988505747126436,0.0,0.0,0.0,0.0
+MONEY,0.0,0.0,0.023809523809523808,0.0,0.0,0.0,0.0,0.0,0.023809523809523808,0.0,0.0,0.0,0.0,0.0,0.8809523809523809,0.023809523809523808,0.0,0.023809523809523808,0.0,0.023809523809523808,0.0
+CURR,0.0,0.0,0.024390243902439025,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.975609756097561,0.0,0.0,0.0,0.0,0.0
+LANGUAGE,0.030303030303030304,0.0,0.0,0.030303030303030304,0.06060606060606061,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.030303030303030304,0.0,0.0,0.8484848484848485,0.0,0.0,0.0,0.0
+PERCENT,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0967741935483871,0.0,0.0,0.0,0.0,0.0,0.03225806451612903,0.0,0.0,0.8709677419354839,0.0,0.0,0.0
+PRODUCT,0.0,0.0,0.0,0.0,0.0,0.05263157894736842,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9473684210526315,0.0,0.0
+QUANTITY,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0
+UNIT,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0

numbers_for_paper/metrics/best/test/low_confidence_correct.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d144dd89b09d64e039362543322df82e35ec6b1c84665818d89f6a597e0a80f3
+size 24180020

numbers_for_paper/metrics/best/test/metrics.csv

@@ -0,0 +1,2 @@
+row_count,correct_count,accuracy,micro_precision,micro_recall,micro_f1,macro_all_labels_precision,macro_all_labels_recall,macro_all_labels_f1,weighted_precision,weighted_recall,weighted_f1,macro_present_labels_precision,macro_present_labels_recall,macro_present_labels_f1,macro_precision_all_labels,macro_recall_all_labels,macro_f1_all_labels,macro_precision_present_labels,macro_recall_present_labels,macro_f1_present_labels,top3_accuracy,top5_accuracy,log_loss,mean_top1_confidence,median_top1_confidence,p05_top1_confidence,p95_top1_confidence,mean_confidence_correct,mean_confidence_incorrect,split,checkpoint,labels
+12600,12340,0.9793650793650793,0.9793650793650793,0.9793650793650793,0.9793650793650793,0.9672581856551346,0.9534679938651157,0.9599200012897606,0.9793254669511197,0.9793650793650793,0.9792974524068463,0.9672581856551346,0.9534679938651157,0.9599200012897606,0.9672581856551346,0.9534679938651157,0.9599200012897606,0.9672581856551346,0.9534679938651157,0.9599200012897606,0.9934920634920635,0.9961904761904762,0.15644530338875345,0.9953899278882004,1.0,0.9995064437389374,1.0,0.9980075920637654,0.8711515589402272,test,best,"['GPE', 'ORG', 'DATE', 'PERS', 'NORP', 'ORDINAL', 'OCC', 'EVENT', 'CARDINAL', 'LOC', 'WEBSITE', 'FAC', 'LAW', 'TIME', 'MONEY', 'CURR', 'LANGUAGE', 'PERCENT', 'PRODUCT', 'QUANTITY', 'UNIT']"

numbers_for_paper/metrics/best/test/metrics.json

@@ -0,0 +1,57 @@
+{
+  "row_count": 12600,
+  "correct_count": 12340,
+  "accuracy": 0.9793650793650793,
+  "micro_precision": 0.9793650793650793,
+  "micro_recall": 0.9793650793650793,
+  "micro_f1": 0.9793650793650793,
+  "macro_all_labels_precision": 0.9672581856551346,
+  "macro_all_labels_recall": 0.9534679938651157,
+  "macro_all_labels_f1": 0.9599200012897606,
+  "weighted_precision": 0.9793254669511197,
+  "weighted_recall": 0.9793650793650793,
+  "weighted_f1": 0.9792974524068463,
+  "macro_present_labels_precision": 0.9672581856551346,
+  "macro_present_labels_recall": 0.9534679938651157,
+  "macro_present_labels_f1": 0.9599200012897606,
+  "macro_precision_all_labels": 0.9672581856551346,
+  "macro_recall_all_labels": 0.9534679938651157,
+  "macro_f1_all_labels": 0.9599200012897606,
+  "macro_precision_present_labels": 0.9672581856551346,
+  "macro_recall_present_labels": 0.9534679938651157,
+  "macro_f1_present_labels": 0.9599200012897606,
+  "top3_accuracy": 0.9934920634920635,
+  "top5_accuracy": 0.9961904761904762,
+  "log_loss": 0.15644530338875345,
+  "mean_top1_confidence": 0.9953899278882004,
+  "median_top1_confidence": 1.0,
+  "p05_top1_confidence": 0.9995064437389374,
+  "p95_top1_confidence": 1.0,
+  "mean_confidence_correct": 0.9980075920637654,
+  "mean_confidence_incorrect": 0.8711515589402272,
+  "split": "test",
+  "checkpoint": "best",
+  "labels": [
+    "GPE",
+    "ORG",
+    "DATE",
+    "PERS",
+    "NORP",
+    "ORDINAL",
+    "OCC",
+    "EVENT",
+    "CARDINAL",
+    "LOC",
+    "WEBSITE",
+    "FAC",
+    "LAW",
+    "TIME",
+    "MONEY",
+    "CURR",
+    "LANGUAGE",
+    "PERCENT",
+    "PRODUCT",
+    "QUANTITY",
+    "UNIT"
+  ]
+}

numbers_for_paper/metrics/best/test/per_class_metrics.csv

@@ -0,0 +1,22 @@
+label,support,predicted_count,true_positives,false_positives,false_negatives,true_negatives,precision,recall,f1,one_vs_rest_accuracy
+GPE,2691,2720,2649,71,42,9838,0.9738970588235294,0.9843924191750278,0.9791166143041952,0.991031746031746
+ORG,2659,2652,2601,51,58,9890,0.9807692307692307,0.9781872884543061,0.9794765580869893,0.9913492063492063
+DATE,1977,1987,1968,19,9,10604,0.9904378459989934,0.9954476479514416,0.992936427850656,0.9977777777777778
+PERS,1071,1075,1053,22,18,11507,0.9795348837209302,0.9831932773109243,0.9813606710158435,0.9968253968253968
+NORP,1021,1023,999,24,22,11555,0.9765395894428153,0.9784524975514202,0.9774951076320939,0.9963492063492063
+ORDINAL,781,782,768,14,13,11805,0.9820971867007673,0.9833546734955185,0.982725527831094,0.9978571428571429
+OCC,770,772,762,10,8,11820,0.9870466321243523,0.9896103896103896,0.9883268482490273,0.9985714285714286
+EVENT,377,368,363,5,14,12218,0.9864130434782609,0.9628647214854111,0.974496644295302,0.9984920634920635
+CARDINAL,372,362,354,8,18,12220,0.9779005524861878,0.9516129032258065,0.9645776566757494,0.9979365079365079
+LOC,239,230,223,7,16,12354,0.9695652173913043,0.9330543933054394,0.9509594882729211,0.9981746031746032
+WEBSITE,148,151,146,5,2,12447,0.9668874172185431,0.9864864864864865,0.9765886287625418,0.9994444444444445
+FAC,129,125,116,9,13,12462,0.928,0.8992248062015504,0.9133858267716536,0.9982539682539683
+LAW,91,91,91,0,0,12509,1.0,1.0,1.0,1.0
+TIME,87,83,76,7,11,12506,0.9156626506024096,0.8735632183908046,0.8941176470588236,0.9985714285714286
+MONEY,42,38,37,1,5,12557,0.9736842105263158,0.8809523809523809,0.925,0.9995238095238095
+CURR,41,41,40,1,1,12558,0.975609756097561,0.975609756097561,0.975609756097561,0.9998412698412699
+LANGUAGE,33,32,28,4,5,12563,0.875,0.8484848484848485,0.8615384615384616,0.9992857142857143
+PERCENT,31,28,27,1,4,12568,0.9642857142857143,0.8709677419354839,0.9152542372881356,0.9996031746031746
+PRODUCT,19,18,18,0,1,12581,1.0,0.9473684210526315,0.972972972972973,0.9999206349206349
+QUANTITY,10,11,10,1,0,12589,0.9090909090909091,1.0,0.9523809523809523,0.9999206349206349
+UNIT,11,11,11,0,0,12589,1.0,1.0,1.0,1.0

numbers_for_paper/metrics/best/test/top_confusions.csv

@@ -0,0 +1,31 @@
+gold,predicted,count
+ORG,GPE,36
+GPE,ORG,21
+LOC,GPE,11
+FAC,GPE,9
+EVENT,ORG,8
+CARDINAL,ORDINAL,8
+NORP,ORG,7
+GPE,NORP,6
+ORG,NORP,6
+ORG,PERS,5
+NORP,PERS,5
+ORDINAL,ORG,5
+GPE,DATE,4
+PERS,GPE,4
+PERS,ORG,4
+PERS,OCC,4
+CARDINAL,DATE,4
+TIME,ORDINAL,4
+GPE,LOC,3
+ORG,EVENT,3
+ORG,FAC,3
+NORP,GPE,3
+ORDINAL,DATE,3
+OCC,ORG,3
+CARDINAL,OCC,3
+PERCENT,CARDINAL,3
+GPE,PERS,2
+GPE,WEBSITE,2
+ORG,LOC,2
+DATE,GPE,2

numbers_for_paper/metrics/best/validation/category_metrics.csv

@@ -0,0 +1,5 @@
+row_count,correct_count,accuracy,micro_precision,micro_recall,micro_f1,macro_all_labels_precision,macro_all_labels_recall,macro_all_labels_f1,weighted_precision,weighted_recall,weighted_f1,macro_present_labels_precision,macro_present_labels_recall,macro_present_labels_f1,macro_precision_all_labels,macro_recall_all_labels,macro_f1_all_labels,macro_precision_present_labels,macro_recall_present_labels,macro_f1_present_labels,top3_accuracy,top5_accuracy,log_loss,mean_top1_confidence,median_top1_confidence,p05_top1_confidence,p95_top1_confidence,mean_confidence_correct,mean_confidence_incorrect,category_field,category_value
+11661,11433,0.9804476459994854,0.9804476459994854,0.9804476459994854,0.9804476459994854,0.9751134735007793,0.9537701979295099,0.963262123278673,0.980497781530163,0.9804476459994854,0.9804033222521132,0.9751134735007793,0.9537701979295099,0.963262123278673,0.9751134735007793,0.9537701979295099,0.963262123278673,0.9751134735007793,0.9537701979295099,0.963262123278673,0.9945116199296802,0.9969985421490438,0.1499927959555766,0.9957854074568397,1.0,0.9996656179428101,1.0,0.998117084670732,0.8788640671654752,evaluation_category,seen_sentence_new_entity
+939,888,0.9456869009584664,0.9456869009584664,0.9456869009584664,0.9456869009584664,0.9014372719333641,0.8429473828005742,0.8655632165863235,0.9458822212425712,0.9456869009584664,0.945010776949412,0.9465091355300324,0.8850947519406029,0.9088413774156396,0.9014372719333641,0.8429473828005742,0.8655632165863235,0.9465091355300324,0.8850947519406029,0.9088413774156396,0.9818956336528222,0.9893503727369543,0.32469566170232955,0.9863015133986712,0.9999997615814209,0.9620923995971681,1.0,0.995362913521292,0.8285265465577444,evaluation_category,unseen_sentence
+12413,12149,0.978731974542818,0.978731974542818,0.978731974542818,0.978731974542818,0.6509775805413648,0.644555559314807,0.6475822917462754,0.9788955393719916,0.978731974542818,0.9787684984983661,0.9764663708120471,0.9668333389722106,0.9713734376194131,0.6509775805413648,0.644555559314807,0.6475822917462754,0.9764663708120471,0.9668333389722106,0.9713734376194131,0.993796825908322,0.996535889792959,0.1575810642972268,0.9952087854720579,1.0,0.9995381832122803,1.0,0.9978953021545451,0.8715781370798746,label_frequency_category,frequent_label_gold_support_ge_500_dataset
+187,172,0.9197860962566845,0.9197860962566845,0.9197860962566845,0.9197860962566845,0.32582972582972586,0.30301034516529984,0.31306520177638597,0.9801652892561983,0.9197860962566845,0.9467421016788319,0.9774891774891775,0.9090310354958995,0.9391956053291579,0.32582972582972586,0.30301034516529984,0.31306520177638597,0.9774891774891775,0.9090310354958995,0.9391956053291579,0.9786096256684492,0.9893048128342246,0.5235372666042232,0.9864390554593846,0.9999951124191284,0.9900969922542572,0.9999997615814209,0.9995631998361543,0.8359488666057586,label_frequency_category,rare_label_gold_support_lt_500_dataset

numbers_for_paper/metrics/best/validation/category_metrics.json

@@ -0,0 +1,142 @@
+{
+  "evaluation_category": {
+    "seen_sentence_new_entity": {
+      "row_count": 11661,
+      "correct_count": 11433,
+      "accuracy": 0.9804476459994854,
+      "micro_precision": 0.9804476459994854,
+      "micro_recall": 0.9804476459994854,
+      "micro_f1": 0.9804476459994854,
+      "macro_all_labels_precision": 0.9751134735007793,
+      "macro_all_labels_recall": 0.9537701979295099,
+      "macro_all_labels_f1": 0.963262123278673,
+      "weighted_precision": 0.980497781530163,
+      "weighted_recall": 0.9804476459994854,
+      "weighted_f1": 0.9804033222521132,
+      "macro_present_labels_precision": 0.9751134735007793,
+      "macro_present_labels_recall": 0.9537701979295099,
+      "macro_present_labels_f1": 0.963262123278673,
+      "macro_precision_all_labels": 0.9751134735007793,
+      "macro_recall_all_labels": 0.9537701979295099,
+      "macro_f1_all_labels": 0.963262123278673,
+      "macro_precision_present_labels": 0.9751134735007793,
+      "macro_recall_present_labels": 0.9537701979295099,
+      "macro_f1_present_labels": 0.963262123278673,
+      "top3_accuracy": 0.9945116199296802,
+      "top5_accuracy": 0.9969985421490438,
+      "log_loss": 0.1499927959555766,
+      "mean_top1_confidence": 0.9957854074568397,
+      "median_top1_confidence": 1.0,
+      "p05_top1_confidence": 0.9996656179428101,
+      "p95_top1_confidence": 1.0,
+      "mean_confidence_correct": 0.998117084670732,
+      "mean_confidence_incorrect": 0.8788640671654752,
+      "category_field": "evaluation_category",
+      "category_value": "seen_sentence_new_entity"
+    },
+    "unseen_sentence": {
+      "row_count": 939,
+      "correct_count": 888,
+      "accuracy": 0.9456869009584664,
+      "micro_precision": 0.9456869009584664,
+      "micro_recall": 0.9456869009584664,
+      "micro_f1": 0.9456869009584664,
+      "macro_all_labels_precision": 0.9014372719333641,
+      "macro_all_labels_recall": 0.8429473828005742,
+      "macro_all_labels_f1": 0.8655632165863235,
+      "weighted_precision": 0.9458822212425712,
+      "weighted_recall": 0.9456869009584664,
+      "weighted_f1": 0.945010776949412,
+      "macro_present_labels_precision": 0.9465091355300324,
+      "macro_present_labels_recall": 0.8850947519406029,
+      "macro_present_labels_f1": 0.9088413774156396,
+      "macro_precision_all_labels": 0.9014372719333641,
+      "macro_recall_all_labels": 0.8429473828005742,
+      "macro_f1_all_labels": 0.8655632165863235,
+      "macro_precision_present_labels": 0.9465091355300324,
+      "macro_recall_present_labels": 0.8850947519406029,
+      "macro_f1_present_labels": 0.9088413774156396,
+      "top3_accuracy": 0.9818956336528222,
+      "top5_accuracy": 0.9893503727369543,
+      "log_loss": 0.32469566170232955,
+      "mean_top1_confidence": 0.9863015133986712,
+      "median_top1_confidence": 0.9999997615814209,
+      "p05_top1_confidence": 0.9620923995971681,
+      "p95_top1_confidence": 1.0,
+      "mean_confidence_correct": 0.995362913521292,
+      "mean_confidence_incorrect": 0.8285265465577444,
+      "category_field": "evaluation_category",
+      "category_value": "unseen_sentence"
+    }
+  },
+  "label_frequency_category": {
+    "frequent_label_gold_support_ge_500_dataset": {
+      "row_count": 12413,
+      "correct_count": 12149,
+      "accuracy": 0.978731974542818,
+      "micro_precision": 0.978731974542818,
+      "micro_recall": 0.978731974542818,
+      "micro_f1": 0.978731974542818,
+      "macro_all_labels_precision": 0.6509775805413648,
+      "macro_all_labels_recall": 0.644555559314807,
+      "macro_all_labels_f1": 0.6475822917462754,
+      "weighted_precision": 0.9788955393719916,
+      "weighted_recall": 0.978731974542818,
+      "weighted_f1": 0.9787684984983661,
+      "macro_present_labels_precision": 0.9764663708120471,
+      "macro_present_labels_recall": 0.9668333389722106,
+      "macro_present_labels_f1": 0.9713734376194131,
+      "macro_precision_all_labels": 0.6509775805413648,
+      "macro_recall_all_labels": 0.644555559314807,
+      "macro_f1_all_labels": 0.6475822917462754,
+      "macro_precision_present_labels": 0.9764663708120471,
+      "macro_recall_present_labels": 0.9668333389722106,
+      "macro_f1_present_labels": 0.9713734376194131,
+      "top3_accuracy": 0.993796825908322,
+      "top5_accuracy": 0.996535889792959,
+      "log_loss": 0.1575810642972268,
+      "mean_top1_confidence": 0.9952087854720579,
+      "median_top1_confidence": 1.0,
+      "p05_top1_confidence": 0.9995381832122803,
+      "p95_top1_confidence": 1.0,
+      "mean_confidence_correct": 0.9978953021545451,
+      "mean_confidence_incorrect": 0.8715781370798746,
+      "category_field": "label_frequency_category",
+      "category_value": "frequent_label_gold_support_ge_500_dataset"
+    },
+    "rare_label_gold_support_lt_500_dataset": {
+      "row_count": 187,
+      "correct_count": 172,
+      "accuracy": 0.9197860962566845,
+      "micro_precision": 0.9197860962566845,
+      "micro_recall": 0.9197860962566845,
+      "micro_f1": 0.9197860962566845,
+      "macro_all_labels_precision": 0.32582972582972586,
+      "macro_all_labels_recall": 0.30301034516529984,
+      "macro_all_labels_f1": 0.31306520177638597,
+      "weighted_precision": 0.9801652892561983,
+      "weighted_recall": 0.9197860962566845,
+      "weighted_f1": 0.9467421016788319,
+      "macro_present_labels_precision": 0.9774891774891775,
+      "macro_present_labels_recall": 0.9090310354958995,
+      "macro_present_labels_f1": 0.9391956053291579,
+      "macro_precision_all_labels": 0.32582972582972586,
+      "macro_recall_all_labels": 0.30301034516529984,
+      "macro_f1_all_labels": 0.31306520177638597,
+      "macro_precision_present_labels": 0.9774891774891775,
+      "macro_recall_present_labels": 0.9090310354958995,
+      "macro_f1_present_labels": 0.9391956053291579,
+      "top3_accuracy": 0.9786096256684492,
+      "top5_accuracy": 0.9893048128342246,
+      "log_loss": 0.5235372666042232,
+      "mean_top1_confidence": 0.9864390554593846,
+      "median_top1_confidence": 0.9999951124191284,
+      "p05_top1_confidence": 0.9900969922542572,
+      "p95_top1_confidence": 0.9999997615814209,
+      "mean_confidence_correct": 0.9995631998361543,
+      "mean_confidence_incorrect": 0.8359488666057586,
+      "category_field": "label_frequency_category",
+      "category_value": "rare_label_gold_support_lt_500_dataset"
+    }
+  }
+}

numbers_for_paper/metrics/best/validation/class_wise_most_common_confusions.csv

@@ -0,0 +1,70 @@
+gold,predicted,count
+GPE,ORG,27
+GPE,PERS,4
+GPE,DATE,3
+GPE,NORP,3
+GPE,EVENT,3
+ORG,GPE,21
+ORG,NORP,5
+ORG,PERS,4
+ORG,EVENT,4
+ORG,FAC,3
+DATE,GPE,3
+DATE,ORDINAL,3
+DATE,PERS,1
+DATE,NORP,1
+DATE,OCC,1
+PERS,GPE,7
+PERS,NORP,7
+PERS,ORG,5
+PERS,OCC,5
+PERS,FAC,3
+NORP,ORG,19
+NORP,PERS,11
+NORP,GPE,7
+NORP,OCC,3
+NORP,EVENT,3
+ORDINAL,NORP,4
+ORDINAL,GPE,3
+ORDINAL,ORG,2
+ORDINAL,PERS,2
+ORDINAL,CARDINAL,2
+OCC,PERS,5
+OCC,GPE,2
+OCC,ORG,1
+OCC,NORP,1
+OCC,WEBSITE,1
+EVENT,ORG,10
+EVENT,GPE,4
+EVENT,DATE,2
+EVENT,NORP,2
+EVENT,FAC,2
+CARDINAL,ORDINAL,6
+CARDINAL,DATE,1
+CARDINAL,TIME,1
+LOC,GPE,5
+LOC,ORG,2
+LOC,NORP,1
+LOC,FAC,1
+WEBSITE,ORG,1
+FAC,GPE,6
+FAC,ORG,2
+FAC,EVENT,1
+TIME,DATE,4
+TIME,GPE,2
+TIME,ORDINAL,2
+TIME,ORG,1
+TIME,LOC,1
+CURR,OCC,1
+CURR,MONEY,1
+LANGUAGE,ORG,2
+LANGUAGE,GPE,1
+LANGUAGE,NORP,1
+LANGUAGE,TIME,1
+PERCENT,CARDINAL,2
+PERCENT,MONEY,1
+PRODUCT,ORG,1
+PRODUCT,PERS,1
+PRODUCT,ORDINAL,1
+QUANTITY,MONEY,1
+QUANTITY,UNIT,1

numbers_for_paper/metrics/best/validation/classification_report.csv

@@ -0,0 +1,25 @@
+,precision,recall,f1-score,support
+GPE,0.977432482426933,0.9821561338289962,0.9797886148711292,2690.0
+ORG,0.9728321548195012,0.9827067669172932,0.9777445296427903,2660.0
+DATE,0.9934310257705912,0.9949392712550608,0.9941845764854614,1976.0
+PERS,0.9738317757009346,0.9720149253731343,0.972922502334267,1072.0
+NORP,0.9750499001996008,0.9559686888454012,0.9654150197628458,1022.0
+ORDINAL,0.9795134443021767,0.9807692307692307,0.9801409352978859,780.0
+OCC,0.981888745148771,0.9869960988296489,0.9844357976653697,769.0
+EVENT,0.9673024523160763,0.9416445623342176,0.9543010752688172,377.0
+CARDINAL,0.9733333333333334,0.9785522788203753,0.9759358288770054,373.0
+LOC,0.9745762711864406,0.9623430962343096,0.968421052631579,239.0
+WEBSITE,0.9865771812080537,0.9932432432432432,0.98989898989899,148.0
+FAC,0.9166666666666666,0.9307692307692308,0.9236641221374046,130.0
+LAW,1.0,1.0,1.0,90.0
+TIME,0.9743589743589743,0.8735632183908046,0.9212121212121213,87.0
+MONEY,0.9333333333333333,1.0,0.9655172413793104,42.0
+CURR,1.0,0.9512195121951219,0.975,41.0
+LANGUAGE,0.9333333333333333,0.8484848484848485,0.8888888888888888,33.0
+PERCENT,1.0,0.9032258064516129,0.9491525423728814,31.0
+PRODUCT,1.0,0.8421052631578947,0.9142857142857143,19.0
+QUANTITY,1.0,0.8181818181818182,0.9,11.0
+UNIT,0.9090909090909091,1.0,0.9523809523809523,10.0
+accuracy,0.9778571428571429,0.9778571428571429,0.9778571428571429,0.9778571428571429
+macro avg,0.9725024753902682,0.9475659044801065,0.9587281193044483,12600.0
+weighted avg,0.977879248064854,0.9778571428571429,0.9777906938936711,12600.0

numbers_for_paper/metrics/best/validation/classification_report.json

@@ -0,0 +1,141 @@
+{
+  "GPE": {
+    "precision": 0.977432482426933,
+    "recall": 0.9821561338289962,
+    "f1-score": 0.9797886148711292,
+    "support": 2690.0
+  },
+  "ORG": {
+    "precision": 0.9728321548195012,
+    "recall": 0.9827067669172932,
+    "f1-score": 0.9777445296427903,
+    "support": 2660.0
+  },
+  "DATE": {
+    "precision": 0.9934310257705912,
+    "recall": 0.9949392712550608,
+    "f1-score": 0.9941845764854614,
+    "support": 1976.0
+  },
+  "PERS": {
+    "precision": 0.9738317757009346,
+    "recall": 0.9720149253731343,
+    "f1-score": 0.972922502334267,
+    "support": 1072.0
+  },
+  "NORP": {
+    "precision": 0.9750499001996008,
+    "recall": 0.9559686888454012,
+    "f1-score": 0.9654150197628458,
+    "support": 1022.0
+  },
+  "ORDINAL": {
+    "precision": 0.9795134443021767,
+    "recall": 0.9807692307692307,
+    "f1-score": 0.9801409352978859,
+    "support": 780.0
+  },
+  "OCC": {
+    "precision": 0.981888745148771,
+    "recall": 0.9869960988296489,
+    "f1-score": 0.9844357976653697,
+    "support": 769.0
+  },
+  "EVENT": {
+    "precision": 0.9673024523160763,
+    "recall": 0.9416445623342176,
+    "f1-score": 0.9543010752688172,
+    "support": 377.0
+  },
+  "CARDINAL": {
+    "precision": 0.9733333333333334,
+    "recall": 0.9785522788203753,
+    "f1-score": 0.9759358288770054,
+    "support": 373.0
+  },
+  "LOC": {
+    "precision": 0.9745762711864406,
+    "recall": 0.9623430962343096,
+    "f1-score": 0.968421052631579,
+    "support": 239.0
+  },
+  "WEBSITE": {
+    "precision": 0.9865771812080537,
+    "recall": 0.9932432432432432,
+    "f1-score": 0.98989898989899,
+    "support": 148.0
+  },
+  "FAC": {
+    "precision": 0.9166666666666666,
+    "recall": 0.9307692307692308,
+    "f1-score": 0.9236641221374046,
+    "support": 130.0
+  },
+  "LAW": {
+    "precision": 1.0,
+    "recall": 1.0,
+    "f1-score": 1.0,
+    "support": 90.0
+  },
+  "TIME": {
+    "precision": 0.9743589743589743,
+    "recall": 0.8735632183908046,
+    "f1-score": 0.9212121212121213,
+    "support": 87.0
+  },
+  "MONEY": {
+    "precision": 0.9333333333333333,
+    "recall": 1.0,
+    "f1-score": 0.9655172413793104,
+    "support": 42.0
+  },
+  "CURR": {
+    "precision": 1.0,
+    "recall": 0.9512195121951219,
+    "f1-score": 0.975,
+    "support": 41.0
+  },
+  "LANGUAGE": {
+    "precision": 0.9333333333333333,
+    "recall": 0.8484848484848485,
+    "f1-score": 0.8888888888888888,
+    "support": 33.0
+  },
+  "PERCENT": {
+    "precision": 1.0,
+    "recall": 0.9032258064516129,
+    "f1-score": 0.9491525423728814,
+    "support": 31.0
+  },
+  "PRODUCT": {
+    "precision": 1.0,
+    "recall": 0.8421052631578947,
+    "f1-score": 0.9142857142857143,
+    "support": 19.0
+  },
+  "QUANTITY": {
+    "precision": 1.0,
+    "recall": 0.8181818181818182,
+    "f1-score": 0.9,
+    "support": 11.0
+  },
+  "UNIT": {
+    "precision": 0.9090909090909091,
+    "recall": 1.0,
+    "f1-score": 0.9523809523809523,
+    "support": 10.0
+  },
+  "accuracy": 0.9778571428571429,
+  "macro avg": {
+    "precision": 0.9725024753902682,
+    "recall": 0.9475659044801065,
+    "f1-score": 0.9587281193044483,
+    "support": 12600.0
+  },
+  "weighted avg": {
+    "precision": 0.977879248064854,
+    "recall": 0.9778571428571429,
+    "f1-score": 0.9777906938936711,
+    "support": 12600.0
+  }
+}

numbers_for_paper/metrics/best/validation/confusion_matrix.csv

@@ -0,0 +1,22 @@
+,GPE,ORG,DATE,PERS,NORP,ORDINAL,OCC,EVENT,CARDINAL,LOC,WEBSITE,FAC,LAW,TIME,MONEY,CURR,LANGUAGE,PERCENT,PRODUCT,QUANTITY,UNIT
+GPE,2642,27,3,4,3,1,1,3,2,3,0,0,0,0,0,0,1,0,0,0,0
+ORG,21,2614,1,4,5,2,2,4,2,2,0,3,0,0,0,0,0,0,0,0,0
+DATE,3,0,1966,1,1,3,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0
+PERS,7,5,1,1042,7,1,5,0,0,0,1,3,0,0,0,0,0,0,0,0,0
+NORP,7,19,0,11,977,0,3,3,1,0,0,1,0,0,0,0,0,0,0,0,0
+ORDINAL,3,2,1,2,4,765,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0
+OCC,2,1,0,5,1,0,759,0,0,0,1,0,0,0,0,0,0,0,0,0,0
+EVENT,4,10,2,0,2,0,1,355,1,0,0,2,0,0,0,0,0,0,0,0,0
+CARDINAL,0,0,1,0,0,6,0,0,365,0,0,0,0,1,0,0,0,0,0,0,0
+LOC,5,2,0,0,1,0,0,0,0,230,0,1,0,0,0,0,0,0,0,0,0
+WEBSITE,0,1,0,0,0,0,0,0,0,0,147,0,0,0,0,0,0,0,0,0,0
+FAC,6,2,0,0,0,0,0,1,0,0,0,121,0,0,0,0,0,0,0,0,0
+LAW,0,0,0,0,0,0,0,0,0,0,0,0,90,0,0,0,0,0,0,0,0
+TIME,2,1,4,0,0,2,0,0,0,1,0,0,0,76,0,0,1,0,0,0,0
+MONEY,0,0,0,0,0,0,0,0,0,0,0,0,0,0,42,0,0,0,0,0,0
+CURR,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,39,0,0,0,0,0
+LANGUAGE,1,2,0,0,1,0,0,0,0,0,0,0,0,1,0,0,28,0,0,0,0
+PERCENT,0,0,0,0,0,0,0,0,2,0,0,0,0,0,1,0,0,28,0,0,0
+PRODUCT,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,16,0,0
+QUANTITY,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,9,1
+UNIT,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10

numbers_for_paper/metrics/best/validation/confusion_matrix_normalized_pred.csv

@@ -0,0 +1,22 @@
+,GPE,ORG,DATE,PERS,NORP,ORDINAL,OCC,EVENT,CARDINAL,LOC,WEBSITE,FAC,LAW,TIME,MONEY,CURR,LANGUAGE,PERCENT,PRODUCT,QUANTITY,UNIT
+GPE,0.977432482426933,0.010048381094157052,0.0015159171298635675,0.003738317757009346,0.0029940119760479044,0.0012804097311139564,0.00129366106080207,0.008174386920980926,0.005333333333333333,0.012711864406779662,0.0,0.0,0.0,0.0,0.0,0.0,0.03333333333333333,0.0,0.0,0.0,0.0
+ORG,0.00776914539400666,0.9728321548195012,0.0005053057099545225,0.003738317757009346,0.00499001996007984,0.002560819462227913,0.00258732212160414,0.010899182561307902,0.005333333333333333,0.00847457627118644,0.0,0.022727272727272728,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+DATE,0.0011098779134295228,0.0,0.9934310257705912,0.0009345794392523365,0.000998003992015968,0.0038412291933418692,0.00129366106080207,0.0027247956403269754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+PERS,0.002589715131335553,0.0018608113137327876,0.0005053057099545225,0.9738317757009346,0.006986027944111776,0.0012804097311139564,0.00646830530401035,0.0,0.0,0.0,0.006711409395973154,0.022727272727272728,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+NORP,0.002589715131335553,0.0070710829921845925,0.0,0.010280373831775701,0.9750499001996008,0.0,0.0038809831824062097,0.008174386920980926,0.0026666666666666666,0.0,0.0,0.007575757575757576,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+ORDINAL,0.0011098779134295228,0.000744324525493115,0.0005053057099545225,0.001869158878504673,0.003992015968063872,0.9795134443021767,0.0,0.0,0.005333333333333333,0.0,0.0,0.007575757575757576,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+OCC,0.0007399186089530152,0.0003721622627465575,0.0,0.004672897196261682,0.000998003992015968,0.0,0.981888745148771,0.0,0.0,0.0,0.006711409395973154,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+EVENT,0.0014798372179060304,0.003721622627465575,0.001010611419909045,0.0,0.001996007984031936,0.0,0.00129366106080207,0.9673024523160763,0.0026666666666666666,0.0,0.0,0.015151515151515152,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+CARDINAL,0.0,0.0,0.0005053057099545225,0.0,0.0,0.0076824583866837385,0.0,0.0,0.9733333333333334,0.0,0.0,0.0,0.0,0.01282051282051282,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+LOC,0.001849796522382538,0.000744324525493115,0.0,0.0,0.000998003992015968,0.0,0.0,0.0,0.0,0.9745762711864406,0.0,0.007575757575757576,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+WEBSITE,0.0,0.0003721622627465575,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9865771812080537,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+FAC,0.0022197558268590455,0.000744324525493115,0.0,0.0,0.0,0.0,0.0,0.0027247956403269754,0.0,0.0,0.0,0.9166666666666666,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+LAW,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+TIME,0.0007399186089530152,0.0003721622627465575,0.00202122283981809,0.0,0.0,0.002560819462227913,0.0,0.0,0.0,0.00423728813559322,0.0,0.0,0.0,0.9743589743589743,0.0,0.0,0.03333333333333333,0.0,0.0,0.0,0.0
+MONEY,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9333333333333333,0.0,0.0,0.0,0.0,0.0,0.0
+CURR,0.0,0.0,0.0,0.0,0.0,0.0,0.00129366106080207,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.022222222222222223,1.0,0.0,0.0,0.0,0.0,0.0
+LANGUAGE,0.0003699593044765076,0.000744324525493115,0.0,0.0,0.000998003992015968,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01282051282051282,0.0,0.0,0.9333333333333333,0.0,0.0,0.0,0.0
+PERCENT,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.005333333333333333,0.0,0.0,0.0,0.0,0.0,0.022222222222222223,0.0,0.0,1.0,0.0,0.0,0.0
+PRODUCT,0.0,0.0003721622627465575,0.0,0.0009345794392523365,0.0,0.0012804097311139564,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0
+QUANTITY,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.022222222222222223,0.0,0.0,0.0,0.0,1.0,0.09090909090909091
+UNIT,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9090909090909091

numbers_for_paper/metrics/best/validation/confusion_matrix_normalized_true.csv

@@ -0,0 +1,22 @@
+,GPE,ORG,DATE,PERS,NORP,ORDINAL,OCC,EVENT,CARDINAL,LOC,WEBSITE,FAC,LAW,TIME,MONEY,CURR,LANGUAGE,PERCENT,PRODUCT,QUANTITY,UNIT
+GPE,0.9821561338289962,0.010037174721189592,0.0011152416356877324,0.001486988847583643,0.0011152416356877324,0.0003717472118959108,0.0003717472118959108,0.0011152416356877324,0.0007434944237918215,0.0011152416356877324,0.0,0.0,0.0,0.0,0.0,0.0,0.0003717472118959108,0.0,0.0,0.0,0.0
+ORG,0.007894736842105263,0.9827067669172932,0.00037593984962406017,0.0015037593984962407,0.0018796992481203006,0.0007518796992481203,0.0007518796992481203,0.0015037593984962407,0.0007518796992481203,0.0007518796992481203,0.0,0.0011278195488721805,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+DATE,0.0015182186234817814,0.0,0.9949392712550608,0.0005060728744939271,0.0005060728744939271,0.0015182186234817814,0.0005060728744939271,0.0005060728744939271,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+PERS,0.0065298507462686565,0.0046641791044776115,0.0009328358208955224,0.9720149253731343,0.0065298507462686565,0.0009328358208955224,0.0046641791044776115,0.0,0.0,0.0,0.0009328358208955224,0.002798507462686567,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+NORP,0.00684931506849315,0.018590998043052837,0.0,0.010763209393346379,0.9559686888454012,0.0,0.0029354207436399216,0.0029354207436399216,0.0009784735812133072,0.0,0.0,0.0009784735812133072,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+ORDINAL,0.0038461538461538464,0.002564102564102564,0.001282051282051282,0.002564102564102564,0.005128205128205128,0.9807692307692307,0.0,0.0,0.002564102564102564,0.0,0.0,0.001282051282051282,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+OCC,0.002600780234070221,0.0013003901170351106,0.0,0.006501950585175552,0.0013003901170351106,0.0,0.9869960988296489,0.0,0.0,0.0,0.0013003901170351106,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+EVENT,0.010610079575596816,0.026525198938992044,0.005305039787798408,0.0,0.005305039787798408,0.0,0.002652519893899204,0.9416445623342176,0.002652519893899204,0.0,0.0,0.005305039787798408,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+CARDINAL,0.0,0.0,0.002680965147453083,0.0,0.0,0.0160857908847185,0.0,0.0,0.9785522788203753,0.0,0.0,0.0,0.0,0.002680965147453083,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+LOC,0.02092050209205021,0.008368200836820083,0.0,0.0,0.0041841004184100415,0.0,0.0,0.0,0.0,0.9623430962343096,0.0,0.0041841004184100415,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+WEBSITE,0.0,0.006756756756756757,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9932432432432432,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+FAC,0.046153846153846156,0.015384615384615385,0.0,0.0,0.0,0.0,0.0,0.007692307692307693,0.0,0.0,0.0,0.9307692307692308,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+LAW,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+TIME,0.022988505747126436,0.011494252873563218,0.04597701149425287,0.0,0.0,0.022988505747126436,0.0,0.0,0.0,0.011494252873563218,0.0,0.0,0.0,0.8735632183908046,0.0,0.0,0.011494252873563218,0.0,0.0,0.0,0.0
+MONEY,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0
+CURR,0.0,0.0,0.0,0.0,0.0,0.0,0.024390243902439025,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.024390243902439025,0.9512195121951219,0.0,0.0,0.0,0.0,0.0
+LANGUAGE,0.030303030303030304,0.06060606060606061,0.0,0.0,0.030303030303030304,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.030303030303030304,0.0,0.0,0.8484848484848485,0.0,0.0,0.0,0.0
+PERCENT,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.06451612903225806,0.0,0.0,0.0,0.0,0.0,0.03225806451612903,0.0,0.0,0.9032258064516129,0.0,0.0,0.0
+PRODUCT,0.0,0.05263157894736842,0.0,0.05263157894736842,0.0,0.05263157894736842,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8421052631578947,0.0,0.0
+QUANTITY,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09090909090909091,0.0,0.0,0.0,0.0,0.8181818181818182,0.09090909090909091
+UNIT,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0

numbers_for_paper/metrics/best/validation/low_confidence_correct.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fd417882d9164bed28a9193492a8a49101d98f90315e2d90b430c2018c084f9
+size 23978971

numbers_for_paper/metrics/best/validation/metrics.csv

@@ -0,0 +1,2 @@
+row_count,correct_count,accuracy,micro_precision,micro_recall,micro_f1,macro_all_labels_precision,macro_all_labels_recall,macro_all_labels_f1,weighted_precision,weighted_recall,weighted_f1,macro_present_labels_precision,macro_present_labels_recall,macro_present_labels_f1,macro_precision_all_labels,macro_recall_all_labels,macro_f1_all_labels,macro_precision_present_labels,macro_recall_present_labels,macro_f1_present_labels,top3_accuracy,top5_accuracy,log_loss,mean_top1_confidence,median_top1_confidence,p05_top1_confidence,p95_top1_confidence,mean_confidence_correct,mean_confidence_incorrect,split,checkpoint,labels
+12600,12321,0.9778571428571429,0.9778571428571429,0.9778571428571429,0.9778571428571429,0.9725024753902682,0.9475659044801065,0.9587281193044483,0.977879248064854,0.9778571428571429,0.9777906938936711,0.9725024753902682,0.9475659044801065,0.9587281193044483,0.9725024753902682,0.9475659044801065,0.9587281193044483,0.9725024753902682,0.9475659044801065,0.9587281193044483,0.9935714285714285,0.9964285714285714,0.16301231904575128,0.9950786315425048,1.0,0.9994885444641113,1.0,0.9979185858491508,0.869662584903847,validation,best,"['GPE', 'ORG', 'DATE', 'PERS', 'NORP', 'ORDINAL', 'OCC', 'EVENT', 'CARDINAL', 'LOC', 'WEBSITE', 'FAC', 'LAW', 'TIME', 'MONEY', 'CURR', 'LANGUAGE', 'PERCENT', 'PRODUCT', 'QUANTITY', 'UNIT']"

numbers_for_paper/metrics/best/validation/metrics.json

@@ -0,0 +1,57 @@
+{
+  "row_count": 12600,
+  "correct_count": 12321,
+  "accuracy": 0.9778571428571429,
+  "micro_precision": 0.9778571428571429,
+  "micro_recall": 0.9778571428571429,
+  "micro_f1": 0.9778571428571429,
+  "macro_all_labels_precision": 0.9725024753902682,
+  "macro_all_labels_recall": 0.9475659044801065,
+  "macro_all_labels_f1": 0.9587281193044483,
+  "weighted_precision": 0.977879248064854,
+  "weighted_recall": 0.9778571428571429,
+  "weighted_f1": 0.9777906938936711,
+  "macro_present_labels_precision": 0.9725024753902682,
+  "macro_present_labels_recall": 0.9475659044801065,
+  "macro_present_labels_f1": 0.9587281193044483,
+  "macro_precision_all_labels": 0.9725024753902682,
+  "macro_recall_all_labels": 0.9475659044801065,
+  "macro_f1_all_labels": 0.9587281193044483,
+  "macro_precision_present_labels": 0.9725024753902682,
+  "macro_recall_present_labels": 0.9475659044801065,
+  "macro_f1_present_labels": 0.9587281193044483,
+  "top3_accuracy": 0.9935714285714285,
+  "top5_accuracy": 0.9964285714285714,
+  "log_loss": 0.16301231904575128,
+  "mean_top1_confidence": 0.9950786315425048,
+  "median_top1_confidence": 1.0,
+  "p05_top1_confidence": 0.9994885444641113,
+  "p95_top1_confidence": 1.0,
+  "mean_confidence_correct": 0.9979185858491508,
+  "mean_confidence_incorrect": 0.869662584903847,
+  "split": "validation",
+  "checkpoint": "best",
+  "labels": [
+    "GPE",
+    "ORG",
+    "DATE",
+    "PERS",
+    "NORP",
+    "ORDINAL",
+    "OCC",
+    "EVENT",
+    "CARDINAL",
+    "LOC",
+    "WEBSITE",
+    "FAC",
+    "LAW",
+    "TIME",
+    "MONEY",
+    "CURR",
+    "LANGUAGE",
+    "PERCENT",
+    "PRODUCT",
+    "QUANTITY",
+    "UNIT"
+  ]
+}

numbers_for_paper/metrics/best/validation/per_class_metrics.csv

@@ -0,0 +1,22 @@
+label,support,predicted_count,true_positives,false_positives,false_negatives,true_negatives,precision,recall,f1,one_vs_rest_accuracy
+GPE,2690,2703,2642,61,48,9849,0.977432482426933,0.9821561338289962,0.9797886148711292,0.9913492063492063
+ORG,2660,2687,2614,73,46,9867,0.9728321548195012,0.9827067669172932,0.9777445296427903,0.9905555555555555
+DATE,1976,1979,1966,13,10,10611,0.9934310257705912,0.9949392712550608,0.9941845764854614,0.9981746031746032
+PERS,1072,1070,1042,28,30,11500,0.9738317757009346,0.9720149253731343,0.972922502334267,0.9953968253968254
+NORP,1022,1002,977,25,45,11553,0.9750499001996008,0.9559686888454012,0.9654150197628458,0.9944444444444445
+ORDINAL,780,781,765,16,15,11804,0.9795134443021767,0.9807692307692307,0.9801409352978859,0.9975396825396825
+OCC,769,773,759,14,10,11817,0.981888745148771,0.9869960988296489,0.9844357976653697,0.9980952380952381
+EVENT,377,367,355,12,22,12211,0.9673024523160763,0.9416445623342176,0.9543010752688172,0.9973015873015874
+CARDINAL,373,375,365,10,8,12217,0.9733333333333334,0.9785522788203753,0.9759358288770054,0.9985714285714286
+LOC,239,236,230,6,9,12355,0.9745762711864406,0.9623430962343096,0.968421052631579,0.9988095238095238
+WEBSITE,148,149,147,2,1,12450,0.9865771812080537,0.9932432432432432,0.98989898989899,0.9997619047619047
+FAC,130,132,121,11,9,12459,0.9166666666666666,0.9307692307692308,0.9236641221374046,0.9984126984126984
+LAW,90,90,90,0,0,12510,1.0,1.0,1.0,1.0
+TIME,87,78,76,2,11,12511,0.9743589743589743,0.8735632183908046,0.9212121212121213,0.998968253968254
+MONEY,42,45,42,3,0,12555,0.9333333333333333,1.0,0.9655172413793104,0.9997619047619047
+CURR,41,39,39,0,2,12559,1.0,0.9512195121951219,0.975,0.9998412698412699
+LANGUAGE,33,30,28,2,5,12565,0.9333333333333333,0.8484848484848485,0.8888888888888888,0.9994444444444445
+PERCENT,31,28,28,0,3,12569,1.0,0.9032258064516129,0.9491525423728814,0.9997619047619047
+PRODUCT,19,16,16,0,3,12581,1.0,0.8421052631578947,0.9142857142857143,0.9997619047619047
+QUANTITY,11,9,9,0,2,12589,1.0,0.8181818181818182,0.9,0.9998412698412699
+UNIT,10,11,10,1,0,12589,0.9090909090909091,1.0,0.9523809523809523,0.9999206349206349

numbers_for_paper/metrics/best/validation/top_confusions.csv

@@ -0,0 +1,31 @@
+gold,predicted,count
+GPE,ORG,27
+ORG,GPE,21
+NORP,ORG,19
+NORP,PERS,11
+EVENT,ORG,10
+PERS,GPE,7
+PERS,NORP,7
+NORP,GPE,7
+CARDINAL,ORDINAL,6
+FAC,GPE,6
+ORG,NORP,5
+PERS,ORG,5
+PERS,OCC,5
+OCC,PERS,5
+LOC,GPE,5
+GPE,PERS,4
+ORG,PERS,4
+ORG,EVENT,4
+ORDINAL,NORP,4
+EVENT,GPE,4
+TIME,DATE,4
+GPE,DATE,3
+GPE,NORP,3
+GPE,EVENT,3
+GPE,LOC,3
+ORG,FAC,3
+DATE,GPE,3
+DATE,ORDINAL,3
+PERS,FAC,3
+NORP,OCC,3

numbers_for_paper/metrics/latest/test/category_metrics.csv

@@ -0,0 +1,5 @@
+row_count,correct_count,accuracy,micro_precision,micro_recall,micro_f1,macro_all_labels_precision,macro_all_labels_recall,macro_all_labels_f1,weighted_precision,weighted_recall,weighted_f1,macro_present_labels_precision,macro_present_labels_recall,macro_present_labels_f1,macro_precision_all_labels,macro_recall_all_labels,macro_f1_all_labels,macro_precision_present_labels,macro_recall_present_labels,macro_f1_present_labels,top3_accuracy,top5_accuracy,log_loss,mean_top1_confidence,median_top1_confidence,p05_top1_confidence,p95_top1_confidence,mean_confidence_correct,mean_confidence_incorrect,category_field,category_value
+11606,11383,0.9807858004480441,0.9807858004480441,0.9807858004480441,0.9807858004480441,0.970248804493168,0.9589518854295287,0.964096406841693,0.9807477003420003,0.9807858004480441,0.9807125672627535,0.970248804493168,0.9589518854295287,0.964096406841693,0.970248804493168,0.9589518854295287,0.964096406841693,0.970248804493168,0.9589518854295287,0.964096406841693,0.9943994485610891,0.9967258314664829,0.14870123961865775,0.995741135430977,1.0,0.9996090829372406,1.0,0.9979960249019697,0.8806406562905675,evaluation_category,seen_sentence_new_entity
+994,957,0.9627766599597586,0.9627766599597586,0.9627766599597586,0.9627766599597586,0.8621871812676783,0.8440708315067772,0.8506415101139764,0.9615439764829642,0.9627766599597586,0.9617803672434098,0.9052965403310622,0.886274373082116,0.8931735856196752,0.8621871812676783,0.8440708315067772,0.8506415101139764,0.9052965403310622,0.886274373082116,0.8931735856196752,0.9818913480885312,0.9899396378269618,0.24393869330234744,0.9909152250954323,0.9999997615814209,0.995781660079956,1.0,0.9978517044798833,0.8115041231786883,evaluation_category,unseen_sentence
+12413,12169,0.9803431885926045,0.9803431885926045,0.9803431885926045,0.9803431885926045,0.6496038901616236,0.64283091403861,0.6461224490926283,0.9806342861930728,0.9803431885926045,0.9804518205405721,0.9744058352424354,0.9642463710579151,0.9691836736389423,0.6496038901616236,0.64283091403861,0.6461224490926283,0.9744058352424354,0.9642463710579151,0.9691836736389423,0.9937162652058326,0.996294207685491,0.1522654708713368,0.9957194130354228,1.0,0.9995974659919739,1.0,0.9981513768838616,0.8744301996270164,label_frequency_category,frequent_label_gold_support_ge_500_dataset
+187,171,0.9144385026737968,0.9144385026737968,0.9144385026737968,0.9144385026737968,0.3248890757143095,0.310637292786805,0.31710717645832703,0.977959909496639,0.9144385026737968,0.9440013868098467,0.9746672271429286,0.9319118783604151,0.9513215293749812,0.3248890757143095,0.310637292786805,0.31710717645832703,0.9746672271429286,0.9319118783604151,0.9513215293749812,0.9732620320855615,0.9893048128342246,0.41834416166187954,0.9715308959790092,0.9999912977218628,0.7683767378330231,0.9999997615814209,0.9861329084599925,0.815471887588501,label_frequency_category,rare_label_gold_support_lt_500_dataset

numbers_for_paper/metrics/latest/test/category_metrics.json

@@ -0,0 +1,142 @@
+{
+  "evaluation_category": {
+    "seen_sentence_new_entity": {
+      "row_count": 11606,
+      "correct_count": 11383,
+      "accuracy": 0.9807858004480441,
+      "micro_precision": 0.9807858004480441,
+      "micro_recall": 0.9807858004480441,
+      "micro_f1": 0.9807858004480441,
+      "macro_all_labels_precision": 0.970248804493168,
+      "macro_all_labels_recall": 0.9589518854295287,
+      "macro_all_labels_f1": 0.964096406841693,
+      "weighted_precision": 0.9807477003420003,
+      "weighted_recall": 0.9807858004480441,
+      "weighted_f1": 0.9807125672627535,
+      "macro_present_labels_precision": 0.970248804493168,
+      "macro_present_labels_recall": 0.9589518854295287,
+      "macro_present_labels_f1": 0.964096406841693,
+      "macro_precision_all_labels": 0.970248804493168,
+      "macro_recall_all_labels": 0.9589518854295287,
+      "macro_f1_all_labels": 0.964096406841693,
+      "macro_precision_present_labels": 0.970248804493168,
+      "macro_recall_present_labels": 0.9589518854295287,
+      "macro_f1_present_labels": 0.964096406841693,
+      "top3_accuracy": 0.9943994485610891,
+      "top5_accuracy": 0.9967258314664829,
+      "log_loss": 0.14870123961865775,
+      "mean_top1_confidence": 0.995741135430977,
+      "median_top1_confidence": 1.0,
+      "p05_top1_confidence": 0.9996090829372406,
+      "p95_top1_confidence": 1.0,
+      "mean_confidence_correct": 0.9979960249019697,
+      "mean_confidence_incorrect": 0.8806406562905675,
+      "category_field": "evaluation_category",
+      "category_value": "seen_sentence_new_entity"
+    },
+    "unseen_sentence": {
+      "row_count": 994,
+      "correct_count": 957,
+      "accuracy": 0.9627766599597586,
+      "micro_precision": 0.9627766599597586,
+      "micro_recall": 0.9627766599597586,
+      "micro_f1": 0.9627766599597586,
+      "macro_all_labels_precision": 0.8621871812676783,
+      "macro_all_labels_recall": 0.8440708315067772,
+      "macro_all_labels_f1": 0.8506415101139764,
+      "weighted_precision": 0.9615439764829642,
+      "weighted_recall": 0.9627766599597586,
+      "weighted_f1": 0.9617803672434098,
+      "macro_present_labels_precision": 0.9052965403310622,
+      "macro_present_labels_recall": 0.886274373082116,
+      "macro_present_labels_f1": 0.8931735856196752,
+      "macro_precision_all_labels": 0.8621871812676783,
+      "macro_recall_all_labels": 0.8440708315067772,
+      "macro_f1_all_labels": 0.8506415101139764,
+      "macro_precision_present_labels": 0.9052965403310622,
+      "macro_recall_present_labels": 0.886274373082116,
+      "macro_f1_present_labels": 0.8931735856196752,
+      "top3_accuracy": 0.9818913480885312,
+      "top5_accuracy": 0.9899396378269618,
+      "log_loss": 0.24393869330234744,
+      "mean_top1_confidence": 0.9909152250954323,
+      "median_top1_confidence": 0.9999997615814209,
+      "p05_top1_confidence": 0.995781660079956,
+      "p95_top1_confidence": 1.0,
+      "mean_confidence_correct": 0.9978517044798833,
+      "mean_confidence_incorrect": 0.8115041231786883,
+      "category_field": "evaluation_category",
+      "category_value": "unseen_sentence"
+    }
+  },
+  "label_frequency_category": {
+    "frequent_label_gold_support_ge_500_dataset": {
+      "row_count": 12413,
+      "correct_count": 12169,
+      "accuracy": 0.9803431885926045,
+      "micro_precision": 0.9803431885926045,
+      "micro_recall": 0.9803431885926045,
+      "micro_f1": 0.9803431885926045,
+      "macro_all_labels_precision": 0.6496038901616236,
+      "macro_all_labels_recall": 0.64283091403861,
+      "macro_all_labels_f1": 0.6461224490926283,
+      "weighted_precision": 0.9806342861930728,
+      "weighted_recall": 0.9803431885926045,
+      "weighted_f1": 0.9804518205405721,
+      "macro_present_labels_precision": 0.9744058352424354,
+      "macro_present_labels_recall": 0.9642463710579151,
+      "macro_present_labels_f1": 0.9691836736389423,
+      "macro_precision_all_labels": 0.6496038901616236,
+      "macro_recall_all_labels": 0.64283091403861,
+      "macro_f1_all_labels": 0.6461224490926283,
+      "macro_precision_present_labels": 0.9744058352424354,
+      "macro_recall_present_labels": 0.9642463710579151,
+      "macro_f1_present_labels": 0.9691836736389423,
+      "top3_accuracy": 0.9937162652058326,
+      "top5_accuracy": 0.996294207685491,
+      "log_loss": 0.1522654708713368,
+      "mean_top1_confidence": 0.9957194130354228,
+      "median_top1_confidence": 1.0,
+      "p05_top1_confidence": 0.9995974659919739,
+      "p95_top1_confidence": 1.0,
+      "mean_confidence_correct": 0.9981513768838616,
+      "mean_confidence_incorrect": 0.8744301996270164,
+      "category_field": "label_frequency_category",
+      "category_value": "frequent_label_gold_support_ge_500_dataset"
+    },
+    "rare_label_gold_support_lt_500_dataset": {
+      "row_count": 187,
+      "correct_count": 171,
+      "accuracy": 0.9144385026737968,
+      "micro_precision": 0.9144385026737968,
+      "micro_recall": 0.9144385026737968,
+      "micro_f1": 0.9144385026737968,
+      "macro_all_labels_precision": 0.3248890757143095,
+      "macro_all_labels_recall": 0.310637292786805,
+      "macro_all_labels_f1": 0.31710717645832703,
+      "weighted_precision": 0.977959909496639,
+      "weighted_recall": 0.9144385026737968,
+      "weighted_f1": 0.9440013868098467,
+      "macro_present_labels_precision": 0.9746672271429286,
+      "macro_present_labels_recall": 0.9319118783604151,
+      "macro_present_labels_f1": 0.9513215293749812,
+      "macro_precision_all_labels": 0.3248890757143095,
+      "macro_recall_all_labels": 0.310637292786805,
+      "macro_f1_all_labels": 0.31710717645832703,
+      "macro_precision_present_labels": 0.9746672271429286,
+      "macro_recall_present_labels": 0.9319118783604151,
+      "macro_f1_present_labels": 0.9513215293749812,
+      "top3_accuracy": 0.9732620320855615,
+      "top5_accuracy": 0.9893048128342246,
+      "log_loss": 0.41834416166187954,
+      "mean_top1_confidence": 0.9715308959790092,
+      "median_top1_confidence": 0.9999912977218628,
+      "p05_top1_confidence": 0.7683767378330231,
+      "p95_top1_confidence": 0.9999997615814209,
+      "mean_confidence_correct": 0.9861329084599925,
+      "mean_confidence_incorrect": 0.815471887588501,
+      "category_field": "label_frequency_category",
+      "category_value": "rare_label_gold_support_lt_500_dataset"
+    }
+  }
+}

numbers_for_paper/metrics/latest/test/class_wise_most_common_confusions.csv

@@ -0,0 +1,74 @@
+gold,predicted,count
+GPE,ORG,21
+GPE,NORP,7
+GPE,DATE,4
+GPE,LOC,3
+GPE,PERS,2
+ORG,GPE,35
+ORG,NORP,6
+ORG,PERS,5
+ORG,EVENT,3
+ORG,FAC,3
+DATE,GPE,2
+DATE,PERS,2
+DATE,TIME,2
+DATE,NORP,1
+DATE,LOC,1
+PERS,GPE,4
+PERS,ORG,4
+PERS,OCC,4
+PERS,DATE,2
+PERS,NORP,2
+NORP,ORG,7
+NORP,PERS,5
+NORP,GPE,3
+NORP,DATE,1
+NORP,OCC,1
+ORDINAL,ORG,5
+ORDINAL,DATE,3
+ORDINAL,CARDINAL,2
+ORDINAL,PERS,1
+ORDINAL,NORP,1
+OCC,ORG,3
+OCC,GPE,1
+OCC,PERS,1
+OCC,NORP,1
+OCC,WEBSITE,1
+EVENT,ORG,8
+EVENT,NORP,2
+EVENT,FAC,2
+EVENT,GPE,1
+EVENT,PERS,1
+CARDINAL,ORDINAL,8
+CARDINAL,DATE,4
+CARDINAL,OCC,3
+CARDINAL,GPE,1
+CARDINAL,PERS,1
+LOC,GPE,11
+LOC,ORG,2
+LOC,NORP,2
+LOC,PERS,1
+WEBSITE,ORG,1
+WEBSITE,TIME,1
+FAC,GPE,9
+FAC,PERS,2
+FAC,DATE,1
+FAC,LOC,1
+TIME,ORDINAL,4
+TIME,GPE,2
+TIME,FAC,2
+TIME,LANGUAGE,2
+TIME,DATE,1
+MONEY,DATE,1
+MONEY,CARDINAL,1
+MONEY,CURR,1
+MONEY,PERCENT,1
+MONEY,QUANTITY,1
+CURR,DATE,1
+LANGUAGE,NORP,2
+LANGUAGE,GPE,1
+LANGUAGE,PERS,1
+LANGUAGE,TIME,1
+PERCENT,CARDINAL,3
+PERCENT,MONEY,1
+PRODUCT,ORDINAL,1

numbers_for_paper/metrics/latest/test/classification_report.csv

@@ -0,0 +1,25 @@
+,precision,recall,f1-score,support
+GPE,0.9742457689477557,0.9840208101077667,0.97910889258643,2691.0
+ORG,0.9807764794572182,0.9785633696878526,0.9796686746987951,2659.0
+DATE,0.9904378459989934,0.9954476479514416,0.992936427850656,1977.0
+PERS,0.9795348837209302,0.9831932773109243,0.9813606710158435,1071.0
+NORP,0.9755859375,0.9784524975514202,0.9770171149144254,1021.0
+ORDINAL,0.9820971867007673,0.9833546734955185,0.982725527831094,781.0
+OCC,0.9870466321243523,0.9896103896103896,0.9883268482490273,770.0
+EVENT,0.9864130434782609,0.9628647214854111,0.974496644295302,377.0
+CARDINAL,0.9779005524861878,0.9516129032258065,0.9645776566757494,372.0
+LOC,0.9695652173913043,0.9330543933054394,0.9509594882729211,239.0
+WEBSITE,0.9668874172185431,0.9864864864864865,0.9765886287625418,148.0
+FAC,0.928,0.8992248062015504,0.9133858267716536,129.0
+LAW,1.0,1.0,1.0,91.0
+TIME,0.9156626506024096,0.8735632183908046,0.8941176470588236,87.0
+MONEY,0.9736842105263158,0.8809523809523809,0.925,42.0
+CURR,0.975609756097561,0.975609756097561,0.975609756097561,41.0
+LANGUAGE,0.875,0.8484848484848485,0.8615384615384616,33.0
+PERCENT,0.9642857142857143,0.8709677419354839,0.9152542372881356,31.0
+PRODUCT,1.0,0.9473684210526315,0.972972972972973,19.0
+QUANTITY,0.9090909090909091,1.0,0.9523809523809523,10.0
+UNIT,1.0,1.0,1.0,11.0
+accuracy,0.9793650793650793,0.9793650793650793,0.9793650793650793,0.9793650793650793
+macro avg,0.9672297240774869,0.953468206825415,0.9599060204410164,12600.0
+weighted avg,0.9793241950917576,0.9793650793650793,0.9792976133880797,12600.0

numbers_for_paper/metrics/latest/test/classification_report.json

@@ -0,0 +1,141 @@
+{
+  "GPE": {
+    "precision": 0.9742457689477557,
+    "recall": 0.9840208101077667,
+    "f1-score": 0.97910889258643,
+    "support": 2691.0
+  },
+  "ORG": {
+    "precision": 0.9807764794572182,
+    "recall": 0.9785633696878526,
+    "f1-score": 0.9796686746987951,
+    "support": 2659.0
+  },
+  "DATE": {
+    "precision": 0.9904378459989934,
+    "recall": 0.9954476479514416,
+    "f1-score": 0.992936427850656,
+    "support": 1977.0
+  },
+  "PERS": {
+    "precision": 0.9795348837209302,
+    "recall": 0.9831932773109243,
+    "f1-score": 0.9813606710158435,
+    "support": 1071.0
+  },
+  "NORP": {
+    "precision": 0.9755859375,
+    "recall": 0.9784524975514202,
+    "f1-score": 0.9770171149144254,
+    "support": 1021.0
+  },
+  "ORDINAL": {
+    "precision": 0.9820971867007673,
+    "recall": 0.9833546734955185,
+    "f1-score": 0.982725527831094,
+    "support": 781.0
+  },
+  "OCC": {
+    "precision": 0.9870466321243523,
+    "recall": 0.9896103896103896,
+    "f1-score": 0.9883268482490273,
+    "support": 770.0
+  },
+  "EVENT": {
+    "precision": 0.9864130434782609,
+    "recall": 0.9628647214854111,
+    "f1-score": 0.974496644295302,
+    "support": 377.0
+  },
+  "CARDINAL": {
+    "precision": 0.9779005524861878,
+    "recall": 0.9516129032258065,
+    "f1-score": 0.9645776566757494,
+    "support": 372.0
+  },
+  "LOC": {
+    "precision": 0.9695652173913043,
+    "recall": 0.9330543933054394,
+    "f1-score": 0.9509594882729211,
+    "support": 239.0
+  },
+  "WEBSITE": {
+    "precision": 0.9668874172185431,
+    "recall": 0.9864864864864865,
+    "f1-score": 0.9765886287625418,
+    "support": 148.0
+  },
+  "FAC": {
+    "precision": 0.928,
+    "recall": 0.8992248062015504,
+    "f1-score": 0.9133858267716536,
+    "support": 129.0
+  },
+  "LAW": {
+    "precision": 1.0,
+    "recall": 1.0,
+    "f1-score": 1.0,
+    "support": 91.0
+  },
+  "TIME": {
+    "precision": 0.9156626506024096,
+    "recall": 0.8735632183908046,
+    "f1-score": 0.8941176470588236,
+    "support": 87.0
+  },
+  "MONEY": {
+    "precision": 0.9736842105263158,
+    "recall": 0.8809523809523809,
+    "f1-score": 0.925,
+    "support": 42.0
+  },
+  "CURR": {
+    "precision": 0.975609756097561,
+    "recall": 0.975609756097561,
+    "f1-score": 0.975609756097561,
+    "support": 41.0
+  },
+  "LANGUAGE": {
+    "precision": 0.875,
+    "recall": 0.8484848484848485,
+    "f1-score": 0.8615384615384616,
+    "support": 33.0
+  },
+  "PERCENT": {
+    "precision": 0.9642857142857143,
+    "recall": 0.8709677419354839,
+    "f1-score": 0.9152542372881356,
+    "support": 31.0
+  },
+  "PRODUCT": {
+    "precision": 1.0,
+    "recall": 0.9473684210526315,
+    "f1-score": 0.972972972972973,
+    "support": 19.0
+  },
+  "QUANTITY": {
+    "precision": 0.9090909090909091,
+    "recall": 1.0,
+    "f1-score": 0.9523809523809523,
+    "support": 10.0
+  },
+  "UNIT": {
+    "precision": 1.0,
+    "recall": 1.0,
+    "f1-score": 1.0,
+    "support": 11.0
+  },
+  "accuracy": 0.9793650793650793,
+  "macro avg": {
+    "precision": 0.9672297240774869,
+    "recall": 0.953468206825415,
+    "f1-score": 0.9599060204410164,
+    "support": 12600.0
+  },
+  "weighted avg": {
+    "precision": 0.9793241950917576,
+    "recall": 0.9793650793650793,
+    "f1-score": 0.9792976133880797,
+    "support": 12600.0
+  }
+}

numbers_for_paper/metrics/latest/test/confusion_matrix.csv

@@ -0,0 +1,22 @@
+,GPE,ORG,DATE,PERS,NORP,ORDINAL,OCC,EVENT,CARDINAL,LOC,WEBSITE,FAC,LAW,TIME,MONEY,CURR,LANGUAGE,PERCENT,PRODUCT,QUANTITY,UNIT
+GPE,2648,21,4,2,7,0,1,1,1,3,2,1,0,0,0,0,0,0,0,0,0
+ORG,35,2602,1,5,6,0,1,3,0,2,1,3,0,0,0,0,0,0,0,0,0
+DATE,2,0,1968,2,1,0,0,0,0,1,0,0,0,2,0,0,1,0,0,0,0
+PERS,4,4,2,1053,2,1,4,0,0,0,1,0,0,0,0,0,0,0,0,0,0
+NORP,3,7,1,5,999,0,1,1,1,0,0,1,0,1,0,0,1,0,0,0,0
+ORDINAL,0,5,3,1,1,768,0,0,2,0,0,0,0,1,0,0,0,0,0,0,0
+OCC,1,3,0,1,1,0,762,0,0,0,1,0,0,1,0,0,0,0,0,0,0
+EVENT,1,8,0,1,2,0,0,363,0,0,0,2,0,0,0,0,0,0,0,0,0
+CARDINAL,1,0,4,1,1,8,3,0,354,0,0,0,0,0,0,0,0,0,0,0,0
+LOC,11,2,0,1,2,0,0,0,0,223,0,0,0,0,0,0,0,0,0,0,0
+WEBSITE,0,1,0,0,0,0,0,0,0,0,146,0,0,1,0,0,0,0,0,0,0
+FAC,9,0,1,2,0,0,0,0,0,1,0,116,0,0,0,0,0,0,0,0,0
+LAW,0,0,0,0,0,0,0,0,0,0,0,0,91,0,0,0,0,0,0,0,0
+TIME,2,0,1,0,0,4,0,0,0,0,0,2,0,76,0,0,2,0,0,0,0
+MONEY,0,0,1,0,0,0,0,0,1,0,0,0,0,0,37,1,0,1,0,1,0
+CURR,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,40,0,0,0,0,0
+LANGUAGE,1,0,0,1,2,0,0,0,0,0,0,0,0,1,0,0,28,0,0,0,0
+PERCENT,0,0,0,0,0,0,0,0,3,0,0,0,0,0,1,0,0,27,0,0,0
+PRODUCT,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,18,0,0
+QUANTITY,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,0
+UNIT,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11