Add m20 weights and results

m20/benchmark_results.json

@@ -0,0 +1,540 @@
+{
+  "model_id": "m20",
+  "checkpoint": "results/m20/final.pt",
+  "trained_step": 15258,
+  "num_fewshot": 0,
+  "limit": null,
+  "results": {
+    "lambada_openai": {
+      "name": "lambada_openai",
+      "alias": "lambada_openai",
+      "sample_len": 5153,
+      "perplexity,none": 575.1581335575003,
+      "perplexity_stderr,none": 31.84994846752592,
+      "acc,none": 0.1847467494663303,
+      "acc_stderr,none": 0.005406880178651611
+    },
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.2711611232822147,
+      "acc_stderr,none": 0.004436505187567018,
+      "acc_norm,none": 0.2804222266480781,
+      "acc_norm_stderr,none": 0.004482874732237671
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.6050054406964092,
+      "acc_stderr,none": 0.011405665187969115,
+      "acc_norm,none": 0.5930359085963003,
+      "acc_norm_stderr,none": 0.011462093919190149
+    },
+    "sciq": {
+      "name": "sciq",
+      "alias": "sciq",
+      "sample_len": 1000,
+      "acc,none": 0.693,
+      "acc_stderr,none": 0.01459328489285274,
+      "acc_norm,none": 0.595,
+      "acc_norm_stderr,none": 0.01553113699045296
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.4524410774410774,
+      "acc_stderr,none": 0.010213265860171308,
+      "acc_norm,none": 0.41035353535353536,
+      "acc_norm_stderr,none": 0.01009353125576533
+    },
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.5430149960536701,
+      "acc_stderr,none": 0.01400038676159808
+    },
+    "blimp_adjunct_island": {
+      "name": "blimp_adjunct_island",
+      "alias": "blimp_adjunct_island",
+      "sample_len": 1000,
+      "acc,none": 0.769,
+      "acc_stderr,none": 0.013334797216936478
+    },
+    "blimp_anaphor_gender_agreement": {
+      "name": "blimp_anaphor_gender_agreement",
+      "alias": "blimp_anaphor_gender_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.921,
+      "acc_stderr,none": 0.00853415677333337
+    },
+    "blimp_anaphor_number_agreement": {
+      "name": "blimp_anaphor_number_agreement",
+      "alias": "blimp_anaphor_number_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.985,
+      "acc_stderr,none": 0.003845749574502972
+    },
+    "blimp_animate_subject_passive": {
+      "name": "blimp_animate_subject_passive",
+      "alias": "blimp_animate_subject_passive",
+      "sample_len": 1000,
+      "acc,none": 0.741,
+      "acc_stderr,none": 0.013860415257527861
+    },
+    "blimp_animate_subject_trans": {
+      "name": "blimp_animate_subject_trans",
+      "alias": "blimp_animate_subject_trans",
+      "sample_len": 1000,
+      "acc,none": 0.902,
+      "acc_stderr,none": 0.009406619184621247
+    },
+    "blimp_causative": {
+      "name": "blimp_causative",
+      "alias": "blimp_causative",
+      "sample_len": 1000,
+      "acc,none": 0.693,
+      "acc_stderr,none": 0.01459328489285274
+    },
+    "blimp_complex_NP_island": {
+      "name": "blimp_complex_NP_island",
+      "alias": "blimp_complex_NP_island",
+      "sample_len": 1000,
+      "acc,none": 0.365,
+      "acc_stderr,none": 0.015231776226264848
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "sample_len": 1000,
+      "acc,none": 0.443,
+      "acc_stderr,none": 0.015716169953204184
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "name": "blimp_coordinate_structure_constraint_object_extraction",
+      "alias": "blimp_coordinate_structure_constraint_object_extraction",
+      "sample_len": 1000,
+      "acc,none": 0.763,
+      "acc_stderr,none": 0.013454070462577935
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "name": "blimp_determiner_noun_agreement_1",
+      "alias": "blimp_determiner_noun_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.972,
+      "acc_stderr,none": 0.005219506034410081
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "name": "blimp_determiner_noun_agreement_2",
+      "alias": "blimp_determiner_noun_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.937,
+      "acc_stderr,none": 0.007687007876286461
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.903,
+      "acc_stderr,none": 0.009363689373248062
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.926,
+      "acc_stderr,none": 0.0082820645127041
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_2",
+      "sample_len": 1000,
+      "acc,none": 0.917,
+      "acc_stderr,none": 0.008728527206074756
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.88,
+      "acc_stderr,none": 0.010281328012747462
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.899,
+      "acc_stderr,none": 0.009533618929341046
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "name": "blimp_determiner_noun_agreement_with_adjective_1",
+      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
+      "sample_len": 1000,
+      "acc,none": 0.947,
+      "acc_stderr,none": 0.00708810561724649
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "name": "blimp_distractor_agreement_relational_noun",
+      "alias": "blimp_distractor_agreement_relational_noun",
+      "sample_len": 1000,
+      "acc,none": 0.85,
+      "acc_stderr,none": 0.011297239823409418
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "name": "blimp_distractor_agreement_relative_clause",
+      "alias": "blimp_distractor_agreement_relative_clause",
+      "sample_len": 1000,
+      "acc,none": 0.72,
+      "acc_stderr,none": 0.014205696104091548
+    },
+    "blimp_drop_argument": {
+      "name": "blimp_drop_argument",
+      "alias": "blimp_drop_argument",
+      "sample_len": 1000,
+      "acc,none": 0.749,
+      "acc_stderr,none": 0.013718133516888775
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "name": "blimp_ellipsis_n_bar_1",
+      "alias": "blimp_ellipsis_n_bar_1",
+      "sample_len": 1000,
+      "acc,none": 0.785,
+      "acc_stderr,none": 0.01299784381903183
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "name": "blimp_ellipsis_n_bar_2",
+      "alias": "blimp_ellipsis_n_bar_2",
+      "sample_len": 1000,
+      "acc,none": 0.843,
+      "acc_stderr,none": 0.011510146979230262
+    },
+    "blimp_existential_there_object_raising": {
+      "name": "blimp_existential_there_object_raising",
+      "alias": "blimp_existential_there_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.799,
+      "acc_stderr,none": 0.012679107214617274
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "name": "blimp_existential_there_quantifiers_1",
+      "alias": "blimp_existential_there_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.957,
+      "acc_stderr,none": 0.006418114379799739
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "name": "blimp_existential_there_quantifiers_2",
+      "alias": "blimp_existential_there_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.173,
+      "acc_stderr,none": 0.011967214137559927
+    },
+    "blimp_existential_there_subject_raising": {
+      "name": "blimp_existential_there_subject_raising",
+      "alias": "blimp_existential_there_subject_raising",
+      "sample_len": 1000,
+      "acc,none": 0.868,
+      "acc_stderr,none": 0.010709373963527965
+    },
+    "blimp_expletive_it_object_raising": {
+      "name": "blimp_expletive_it_object_raising",
+      "alias": "blimp_expletive_it_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.751,
+      "acc_stderr,none": 0.013681600278702275
+    },
+    "blimp_inchoative": {
+      "name": "blimp_inchoative",
+      "alias": "blimp_inchoative",
+      "sample_len": 1000,
+      "acc,none": 0.662,
+      "acc_stderr,none": 0.014965960710224385
+    },
+    "blimp_intransitive": {
+      "name": "blimp_intransitive",
+      "alias": "blimp_intransitive",
+      "sample_len": 1000,
+      "acc,none": 0.769,
+      "acc_stderr,none": 0.013334797216936478
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "name": "blimp_irregular_past_participle_adjectives",
+      "alias": "blimp_irregular_past_participle_adjectives",
+      "sample_len": 1000,
+      "acc,none": 0.955,
+      "acc_stderr,none": 0.006558812241406063
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "name": "blimp_irregular_past_participle_verbs",
+      "alias": "blimp_irregular_past_participle_verbs",
+      "sample_len": 1000,
+      "acc,none": 0.852,
+      "acc_stderr,none": 0.011234866364235145
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_1",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.886,
+      "acc_stderr,none": 0.010055103435823278
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_2",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.853,
+      "acc_stderr,none": 0.011203415395160376
+    },
+    "blimp_left_branch_island_echo_question": {
+      "name": "blimp_left_branch_island_echo_question",
+      "alias": "blimp_left_branch_island_echo_question",
+      "sample_len": 1000,
+      "acc,none": 0.53,
+      "acc_stderr,none": 0.015790799515836725
+    },
+    "blimp_left_branch_island_simple_question": {
+      "name": "blimp_left_branch_island_simple_question",
+      "alias": "blimp_left_branch_island_simple_question",
+      "sample_len": 1000,
+      "acc,none": 0.445,
+      "acc_stderr,none": 0.015723301886761007
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "name": "blimp_matrix_question_npi_licensor_present",
+      "alias": "blimp_matrix_question_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.619,
+      "acc_stderr,none": 0.015364734787007391
+    },
+    "blimp_npi_present_1": {
+      "name": "blimp_npi_present_1",
+      "alias": "blimp_npi_present_1",
+      "sample_len": 1000,
+      "acc,none": 0.49,
+      "acc_stderr,none": 0.015816135752773183
+    },
+    "blimp_npi_present_2": {
+      "name": "blimp_npi_present_2",
+      "alias": "blimp_npi_present_2",
+      "sample_len": 1000,
+      "acc,none": 0.643,
+      "acc_stderr,none": 0.01515852172148659
+    },
+    "blimp_only_npi_licensor_present": {
+      "name": "blimp_only_npi_licensor_present",
+      "alias": "blimp_only_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.858,
+      "acc_stderr,none": 0.011043457699378305
+    },
+    "blimp_only_npi_scope": {
+      "name": "blimp_only_npi_scope",
+      "alias": "blimp_only_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.611,
+      "acc_stderr,none": 0.015424555647308514
+    },
+    "blimp_passive_1": {
+      "name": "blimp_passive_1",
+      "alias": "blimp_passive_1",
+      "sample_len": 1000,
+      "acc,none": 0.889,
+      "acc_stderr,none": 0.009938701010583716
+    },
+    "blimp_passive_2": {
+      "name": "blimp_passive_2",
+      "alias": "blimp_passive_2",
+      "sample_len": 1000,
+      "acc,none": 0.875,
+      "acc_stderr,none": 0.010463483381956722
+    },
+    "blimp_principle_A_c_command": {
+      "name": "blimp_principle_A_c_command",
+      "alias": "blimp_principle_A_c_command",
+      "sample_len": 1000,
+      "acc,none": 0.595,
+      "acc_stderr,none": 0.01553113699045296
+    },
+    "blimp_principle_A_case_1": {
+      "name": "blimp_principle_A_case_1",
+      "alias": "blimp_principle_A_case_1",
+      "sample_len": 1000,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "blimp_principle_A_case_2": {
+      "name": "blimp_principle_A_case_2",
+      "alias": "blimp_principle_A_case_2",
+      "sample_len": 1000,
+      "acc,none": 0.929,
+      "acc_stderr,none": 0.008125578442487959
+    },
+    "blimp_principle_A_domain_1": {
+      "name": "blimp_principle_A_domain_1",
+      "alias": "blimp_principle_A_domain_1",
+      "sample_len": 1000,
+      "acc,none": 0.945,
+      "acc_stderr,none": 0.0072129762946392
+    },
+    "blimp_principle_A_domain_2": {
+      "name": "blimp_principle_A_domain_2",
+      "alias": "blimp_principle_A_domain_2",
+      "sample_len": 1000,
+      "acc,none": 0.723,
+      "acc_stderr,none": 0.014158794845306273
+    },
+    "blimp_principle_A_domain_3": {
+      "name": "blimp_principle_A_domain_3",
+      "alias": "blimp_principle_A_domain_3",
+      "sample_len": 1000,
+      "acc,none": 0.598,
+      "acc_stderr,none": 0.015512467135714959
+    },
+    "blimp_principle_A_reconstruction": {
+      "name": "blimp_principle_A_reconstruction",
+      "alias": "blimp_principle_A_reconstruction",
+      "sample_len": 1000,
+      "acc,none": 0.38,
+      "acc_stderr,none": 0.015356947477797658
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "name": "blimp_regular_plural_subject_verb_agreement_1",
+      "alias": "blimp_regular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.892,
+      "acc_stderr,none": 0.009820001651345708
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "name": "blimp_regular_plural_subject_verb_agreement_2",
+      "alias": "blimp_regular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.881,
+      "acc_stderr,none": 0.01024421514533661
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "name": "blimp_sentential_negation_npi_licensor_present",
+      "alias": "blimp_sentential_negation_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.997,
+      "acc_stderr,none": 0.0017303161543469432
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "name": "blimp_sentential_negation_npi_scope",
+      "alias": "blimp_sentential_negation_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.555,
+      "acc_stderr,none": 0.015723301886761007
+    },
+    "blimp_sentential_subject_island": {
+      "name": "blimp_sentential_subject_island",
+      "alias": "blimp_sentential_subject_island",
+      "sample_len": 1000,
+      "acc,none": 0.384,
+      "acc_stderr,none": 0.015387682761896958
+    },
+    "blimp_superlative_quantifiers_1": {
+      "name": "blimp_superlative_quantifiers_1",
+      "alias": "blimp_superlative_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.924,
+      "acc_stderr,none": 0.008384169266796462
+    },
+    "blimp_superlative_quantifiers_2": {
+      "name": "blimp_superlative_quantifiers_2",
+      "alias": "blimp_superlative_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.819,
+      "acc_stderr,none": 0.012181436179178027
+    },
+    "blimp_tough_vs_raising_1": {
+      "name": "blimp_tough_vs_raising_1",
+      "alias": "blimp_tough_vs_raising_1",
+      "sample_len": 1000,
+      "acc,none": 0.641,
+      "acc_stderr,none": 0.01517726422479853
+    },
+    "blimp_tough_vs_raising_2": {
+      "name": "blimp_tough_vs_raising_2",
+      "alias": "blimp_tough_vs_raising_2",
+      "sample_len": 1000,
+      "acc,none": 0.845,
+      "acc_stderr,none": 0.011450157470799522
+    },
+    "blimp_transitive": {
+      "name": "blimp_transitive",
+      "alias": "blimp_transitive",
+      "sample_len": 1000,
+      "acc,none": 0.804,
+      "acc_stderr,none": 0.012559527926707347
+    },
+    "blimp_wh_island": {
+      "name": "blimp_wh_island",
+      "alias": "blimp_wh_island",
+      "sample_len": 1000,
+      "acc,none": 0.67,
+      "acc_stderr,none": 0.014876872027456623
+    },
+    "blimp_wh_questions_object_gap": {
+      "name": "blimp_wh_questions_object_gap",
+      "alias": "blimp_wh_questions_object_gap",
+      "sample_len": 1000,
+      "acc,none": 0.715,
+      "acc_stderr,none": 0.014282120955200562
+    },
+    "blimp_wh_questions_subject_gap": {
+      "name": "blimp_wh_questions_subject_gap",
+      "alias": "blimp_wh_questions_subject_gap",
+      "sample_len": 1000,
+      "acc,none": 0.921,
+      "acc_stderr,none": 0.00853415677333337
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "name": "blimp_wh_questions_subject_gap_long_distance",
+      "alias": "blimp_wh_questions_subject_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.873,
+      "acc_stderr,none": 0.010534798620855644
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "name": "blimp_wh_vs_that_no_gap",
+      "alias": "blimp_wh_vs_that_no_gap",
+      "sample_len": 1000,
+      "acc,none": 0.963,
+      "acc_stderr,none": 0.0059721576223896395
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "name": "blimp_wh_vs_that_no_gap_long_distance",
+      "alias": "blimp_wh_vs_that_no_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.955,
+      "acc_stderr,none": 0.006558812241406063
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "name": "blimp_wh_vs_that_with_gap",
+      "alias": "blimp_wh_vs_that_with_gap",
+      "sample_len": 1000,
+      "acc,none": 0.512,
+      "acc_stderr,none": 0.015814743314581686
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "name": "blimp_wh_vs_that_with_gap_long_distance",
+      "alias": "blimp_wh_vs_that_with_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.283,
+      "acc_stderr,none": 0.01425181090648182
+    },
+    "blimp": {
+      "alias": "blimp",
+      "name": "blimp",
+      "sample_len": 67000,
+      "acc,none": 0.7641641791044775,
+      "acc_stderr,none": 0.0014648618904583065,
+      "sample_count": {
+        "acc,none": 67000
+      }
+    }
+  }
+}

m20/config.json

@@ -0,0 +1,49 @@
+{
+  "model_id": "m20",
+  "run_name": "m20-mla-rope-latent256-dense-adamw",
+  "data_dir": "data/fineweb-edu",
+  "vocab_size": 50257,
+  "block_size": 1024,
+  "d_model": 512,
+  "num_layers": 12,
+  "num_heads": 8,
+  "ffn_mult": 4,
+  "dropout": 0.0,
+  "bias": false,
+  "norm_type": "rmsnorm",
+  "tie_embeddings": true,
+  "attention_type": "mla",
+  "gqa_num_kv_heads": 2,
+  "mla_latent_dim": 256,
+  "pos_type": "rope",
+  "rope_base": 10000,
+  "ffn_type": "dense",
+  "moe_num_experts": 8,
+  "moe_top_k": 2,
+  "optimizer": "adamw",
+  "lr": 0.0006,
+  "min_lr": 6e-05,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "muon_lr": 0.02,
+  "muon_momentum": 0.95,
+  "training_tokens": 1000000000,
+  "warmup_tokens": 6000000,
+  "lr_schedule": "cosine",
+  "batch_size": 4,
+  "grad_accum_steps": 16,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "seed": 1337,
+  "eval_interval": 250,
+  "eval_iters": 100,
+  "log_interval": 10,
+  "checkpoint_interval": 1000,
+  "results_dir": "results",
+  "mlflow_experiment": "slm-ablation",
+  "mlflow_tracking_uri": "sqlite:///mlflow.db",
+  "mlflow_artifact_location": "./mlartifacts"
+}

m20/eval_results.json

@@ -0,0 +1,212 @@
+{
+  "model_id": "m20",
+  "checkpoint": "/home/akshat/codemindv2/results/m20/final.pt",
+  "trained_step": 15258,
+  "total_params": 61920256,
+  "active_params": 61920256,
+  "flops_per_forward": 123840512.0,
+  "val_loss": 3.4625886261463164,
+  "val_perplexity": 31.899445466560927,
+  "bits_per_token": 4.99545943957982,
+  "ppl_tokens_evaluated": 655360,
+  "prefill_tok_per_sec": 158358.59954810305,
+  "inference_tok_per_sec": 178.26855692268356,
+  "inference_tok_per_sec_kvcache": 191.98823619444323,
+  "kvcache_speedup": 1.076960735581149,
+  "gen_tokens": 128,
+  "peak_infer_vram_gb": 0.385,
+  "peak_train_vram_gb": 4.062,
+  "closeness_answer_ppl": 23.819,
+  "closeness_hit_rate": 0.45,
+  "closeness_token_f1": 0.014,
+  "closeness_n": 20,
+  "closeness_items": [
+    {
+      "prompt": "The capital of France is",
+      "answer": " Paris",
+      "generated": " the city of Paris.\n",
+      "answer_nll": 2.9506,
+      "answer_ppl": 19.117,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The capital of Japan is",
+      "answer": " Tokyo",
+      "generated": " the capital of the Republic of",
+      "answer_nll": 2.6959,
+      "answer_ppl": 14.818,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The largest planet in the solar system is",
+      "answer": " Jupiter",
+      "generated": " the Sun, which is the",
+      "answer_nll": 2.8111,
+      "answer_ppl": 16.628,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Water is made of hydrogen and",
+      "answer": " oxygen",
+      "generated": " oxygen.\nThe hydrogen is",
+      "answer_nll": 0.7412,
+      "answer_ppl": 2.099,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "answer": " cold",
+      "generated": " the case for the two-",
+      "answer_nll": 5.2195,
+      "answer_ppl": 184.842,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of up is",
+      "answer": " down",
+      "generated": " the case of the two-",
+      "answer_nll": 7.8645,
+      "answer_ppl": 2603.199,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Two plus two equals",
+      "answer": " four",
+      "generated": " one.\nThe first two",
+      "answer_nll": 3.4823,
+      "answer_ppl": 32.534,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The sun rises in the",
+      "answer": " east",
+      "generated": " sky, and the sun rises",
+      "answer_nll": 3.5493,
+      "answer_ppl": 34.79,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The first president of the United States was George",
+      "answer": " Washington",
+      "generated": " Washington.\nThe first president",
+      "answer_nll": 0.3028,
+      "answer_ppl": 1.354,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The speed of light is approximately 300,000 kilometers per",
+      "answer": " second",
+      "generated": " second.\nThe speed of",
+      "answer_nll": 0.3426,
+      "answer_ppl": 1.409,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "A triangle has three",
+      "answer": " sides",
+      "generated": " sides.\n- The triangle",
+      "answer_nll": 1.2375,
+      "answer_ppl": 3.447,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The chemical symbol for gold is",
+      "answer": " Au",
+      "generated": " gold.\nThe gold symbol",
+      "answer_nll": 9.1208,
+      "answer_ppl": 9143.113,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The freezing point of water in Celsius is",
+      "answer": " zero",
+      "generated": " about 1.5 degrees Celsius",
+      "answer_nll": 4.3002,
+      "answer_ppl": 73.711,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The author of Romeo and Juliet is William",
+      "answer": " Shakespeare",
+      "generated": " Shakespeare.\nThe author of",
+      "answer_nll": 0.1833,
+      "answer_ppl": 1.201,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The Earth revolves around the",
+      "answer": " Sun",
+      "generated": " sun.\nThe Earth is",
+      "answer_nll": 0.9723,
+      "answer_ppl": 2.644,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The plural of mouse is",
+      "answer": " mice",
+      "generated": " the plural of mouse.\n",
+      "answer_nll": 7.0402,
+      "answer_ppl": 1141.636,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Photosynthesis occurs in the",
+      "answer": " leaves",
+      "generated": " form of photosynthesis.\n",
+      "answer_nll": 3.8898,
+      "answer_ppl": 48.901,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The human heart pumps",
+      "answer": " blood",
+      "generated": " blood and oxygen to the brain",
+      "answer_nll": 1.0091,
+      "answer_ppl": 2.743,
+      "hit": true,
+      "token_f1": 0.286
+    },
+    {
+      "prompt": "The currency used in the United States is the",
+      "answer": " dollar",
+      "generated": " United States dollar.\nThe",
+      "answer_nll": 2.9926,
+      "answer_ppl": 19.938,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The tallest animal in the world is the",
+      "answer": " giraffe",
+      "generated": " elephant.\nThe elephant is",
+      "answer_nll": 2.7043,
+      "answer_ppl": 14.944,
+      "hit": false,
+      "token_f1": 0.0
+    }
+  ],
+  "generation_samples": {
+    "The most important thing about sleep is": "The most important thing about sleep is to look at your own body and how you feel.\n- Try to get out of bed every night.\n- Do you feel sleepy all the time?\n- Do you feel like you are having a bad day, no matter what the time?\n- Do you feel as though you can",
+    "Scientists have discovered that": "Scientists have discovered that, in a recent study of about 30,000 participants from the Australian and European Union, the amount of sunlight in the atmosphere of the planet is about one-third of the sun\u2019s mass.\n\u201cWe are seeing significant changes in the intensity of light, but the effects this type",
+    "In the history of mathematics,": "In the history of mathematics, the term \"physics\" was used to describe the operations of many different fields.\nThe science of physics has become more and more popular in recent years, with the number of scientific journals running, and the number of experimental physicists working on them. There are numerous other fields, and their roles vary",
+    "Python code for prime number": "Python code for prime number of integers.\n- The integer values of the integers for integers are integers and integers.\n- When the integers are integers, the integers are integers and integers.\n- The integer values of the integers are integers and integers.\n- The integers are integers\n- The integers are integers\n-",
+    "The best way to learn a new language is": "The best way to learn a new language is to learn about its features and benefits. It is especially important to study the language in a way that is appropriate for your child's age.\nIt is very important to keep in mind that the best way to learn a language is to learn some basic elements. For example, learn to write a sentence"
+  }
+}

m20/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65703dd98a9c3afa608d2a7d09a83363b4f8cb7bf29c1a2c56826aee4b61d468
+size 247691704