Upload folder using huggingface_hub

m03/benchmark_results.json

@@ -0,0 +1,540 @@
+{
+  "model_id": "m03",
+  "checkpoint": "results/m03/final.pt",
+  "trained_step": 15258,
+  "num_fewshot": 0,
+  "limit": null,
+  "results": {
+    "lambada_openai": {
+      "name": "lambada_openai",
+      "alias": "lambada_openai",
+      "sample_len": 5153,
+      "perplexity,none": 1016.5617475182925,
+      "perplexity_stderr,none": 55.73180004169432,
+      "acc,none": 0.15408499902969144,
+      "acc_stderr,none": 0.00502985674458647
+    },
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.27066321449910374,
+      "acc_stderr,none": 0.004433943894764314,
+      "acc_norm,none": 0.2703644692292372,
+      "acc_norm_stderr,none": 0.004432403734882474
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.5930359085963003,
+      "acc_stderr,none": 0.011462093919190149,
+      "acc_norm,none": 0.5837867247007617,
+      "acc_norm_stderr,none": 0.011500864675166542
+    },
+    "sciq": {
+      "name": "sciq",
+      "alias": "sciq",
+      "sample_len": 1000,
+      "acc,none": 0.679,
+      "acc_stderr,none": 0.01477082181793475,
+      "acc_norm,none": 0.62,
+      "acc_norm_stderr,none": 0.015356947477797658
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.45075757575757575,
+      "acc_stderr,none": 0.01020990610101116,
+      "acc_norm,none": 0.3998316498316498,
+      "acc_norm_stderr,none": 0.010051788039412915
+    },
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.5130228887134964,
+      "acc_stderr,none": 0.014047718393997596
+    },
+    "blimp_adjunct_island": {
+      "name": "blimp_adjunct_island",
+      "alias": "blimp_adjunct_island",
+      "sample_len": 1000,
+      "acc,none": 0.833,
+      "acc_stderr,none": 0.011800434324644593
+    },
+    "blimp_anaphor_gender_agreement": {
+      "name": "blimp_anaphor_gender_agreement",
+      "alias": "blimp_anaphor_gender_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.912,
+      "acc_stderr,none": 0.008963053962592012
+    },
+    "blimp_anaphor_number_agreement": {
+      "name": "blimp_anaphor_number_agreement",
+      "alias": "blimp_anaphor_number_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.978,
+      "acc_stderr,none": 0.004640855259274689
+    },
+    "blimp_animate_subject_passive": {
+      "name": "blimp_animate_subject_passive",
+      "alias": "blimp_animate_subject_passive",
+      "sample_len": 1000,
+      "acc,none": 0.732,
+      "acc_stderr,none": 0.014013292702729519
+    },
+    "blimp_animate_subject_trans": {
+      "name": "blimp_animate_subject_trans",
+      "alias": "blimp_animate_subject_trans",
+      "sample_len": 1000,
+      "acc,none": 0.907,
+      "acc_stderr,none": 0.009188875634996669
+    },
+    "blimp_causative": {
+      "name": "blimp_causative",
+      "alias": "blimp_causative",
+      "sample_len": 1000,
+      "acc,none": 0.701,
+      "acc_stderr,none": 0.014484778521220433
+    },
+    "blimp_complex_NP_island": {
+      "name": "blimp_complex_NP_island",
+      "alias": "blimp_complex_NP_island",
+      "sample_len": 1000,
+      "acc,none": 0.544,
+      "acc_stderr,none": 0.01575792855397918
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "sample_len": 1000,
+      "acc,none": 0.404,
+      "acc_stderr,none": 0.015524980677122437
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "name": "blimp_coordinate_structure_constraint_object_extraction",
+      "alias": "blimp_coordinate_structure_constraint_object_extraction",
+      "sample_len": 1000,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.012655439943366653
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "name": "blimp_determiner_noun_agreement_1",
+      "alias": "blimp_determiner_noun_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.966,
+      "acc_stderr,none": 0.0057338361396953854
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "name": "blimp_determiner_noun_agreement_2",
+      "alias": "blimp_determiner_noun_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.943,
+      "acc_stderr,none": 0.007335175853706905
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.867,
+      "acc_stderr,none": 0.010743669132397295
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.924,
+      "acc_stderr,none": 0.008384169266796462
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_2",
+      "sample_len": 1000,
+      "acc,none": 0.908,
+      "acc_stderr,none": 0.009144376393151129
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.815,
+      "acc_stderr,none": 0.012285191326386656
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.879,
+      "acc_stderr,none": 0.010318210380946179
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "name": "blimp_determiner_noun_agreement_with_adjective_1",
+      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
+      "sample_len": 1000,
+      "acc,none": 0.93,
+      "acc_stderr,none": 0.008072494358323525
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "name": "blimp_distractor_agreement_relational_noun",
+      "alias": "blimp_distractor_agreement_relational_noun",
+      "sample_len": 1000,
+      "acc,none": 0.776,
+      "acc_stderr,none": 0.013190830072364589
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "name": "blimp_distractor_agreement_relative_clause",
+      "alias": "blimp_distractor_agreement_relative_clause",
+      "sample_len": 1000,
+      "acc,none": 0.629,
+      "acc_stderr,none": 0.015283736211823096
+    },
+    "blimp_drop_argument": {
+      "name": "blimp_drop_argument",
+      "alias": "blimp_drop_argument",
+      "sample_len": 1000,
+      "acc,none": 0.729,
+      "acc_stderr,none": 0.014062601350986125
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "name": "blimp_ellipsis_n_bar_1",
+      "alias": "blimp_ellipsis_n_bar_1",
+      "sample_len": 1000,
+      "acc,none": 0.766,
+      "acc_stderr,none": 0.01339490288966006
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "name": "blimp_ellipsis_n_bar_2",
+      "alias": "blimp_ellipsis_n_bar_2",
+      "sample_len": 1000,
+      "acc,none": 0.818,
+      "acc_stderr,none": 0.0122075806376622
+    },
+    "blimp_existential_there_object_raising": {
+      "name": "blimp_existential_there_object_raising",
+      "alias": "blimp_existential_there_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.759,
+      "acc_stderr,none": 0.01353152253451555
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "name": "blimp_existential_there_quantifiers_1",
+      "alias": "blimp_existential_there_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.979,
+      "acc_stderr,none": 0.0045364721513065165
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "name": "blimp_existential_there_quantifiers_2",
+      "alias": "blimp_existential_there_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.17,
+      "acc_stderr,none": 0.011884495834541793
+    },
+    "blimp_existential_there_subject_raising": {
+      "name": "blimp_existential_there_subject_raising",
+      "alias": "blimp_existential_there_subject_raising",
+      "sample_len": 1000,
+      "acc,none": 0.859,
+      "acc_stderr,none": 0.01101091459599248
+    },
+    "blimp_expletive_it_object_raising": {
+      "name": "blimp_expletive_it_object_raising",
+      "alias": "blimp_expletive_it_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.744,
+      "acc_stderr,none": 0.013807775152234145
+    },
+    "blimp_inchoative": {
+      "name": "blimp_inchoative",
+      "alias": "blimp_inchoative",
+      "sample_len": 1000,
+      "acc,none": 0.66,
+      "acc_stderr,none": 0.014987482264363972
+    },
+    "blimp_intransitive": {
+      "name": "blimp_intransitive",
+      "alias": "blimp_intransitive",
+      "sample_len": 1000,
+      "acc,none": 0.75,
+      "acc_stderr,none": 0.013699915608779773
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "name": "blimp_irregular_past_participle_adjectives",
+      "alias": "blimp_irregular_past_participle_adjectives",
+      "sample_len": 1000,
+      "acc,none": 0.891,
+      "acc_stderr,none": 0.009859828407037124
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "name": "blimp_irregular_past_participle_verbs",
+      "alias": "blimp_irregular_past_participle_verbs",
+      "sample_len": 1000,
+      "acc,none": 0.869,
+      "acc_stderr,none": 0.0106748748448379
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_1",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.874,
+      "acc_stderr,none": 0.010499249222407964
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_2",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.844,
+      "acc_stderr,none": 0.011480235006122295
+    },
+    "blimp_left_branch_island_echo_question": {
+      "name": "blimp_left_branch_island_echo_question",
+      "alias": "blimp_left_branch_island_echo_question",
+      "sample_len": 1000,
+      "acc,none": 0.495,
+      "acc_stderr,none": 0.015818508944436743
+    },
+    "blimp_left_branch_island_simple_question": {
+      "name": "blimp_left_branch_island_simple_question",
+      "alias": "blimp_left_branch_island_simple_question",
+      "sample_len": 1000,
+      "acc,none": 0.5,
+      "acc_stderr,none": 0.015819299929208316
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "name": "blimp_matrix_question_npi_licensor_present",
+      "alias": "blimp_matrix_question_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.445,
+      "acc_stderr,none": 0.015723301886761007
+    },
+    "blimp_npi_present_1": {
+      "name": "blimp_npi_present_1",
+      "alias": "blimp_npi_present_1",
+      "sample_len": 1000,
+      "acc,none": 0.558,
+      "acc_stderr,none": 0.01571250721186415
+    },
+    "blimp_npi_present_2": {
+      "name": "blimp_npi_present_2",
+      "alias": "blimp_npi_present_2",
+      "sample_len": 1000,
+      "acc,none": 0.678,
+      "acc_stderr,none": 0.014782913600996745
+    },
+    "blimp_only_npi_licensor_present": {
+      "name": "blimp_only_npi_licensor_present",
+      "alias": "blimp_only_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.663,
+      "acc_stderr,none": 0.01495508791865346
+    },
+    "blimp_only_npi_scope": {
+      "name": "blimp_only_npi_scope",
+      "alias": "blimp_only_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.443,
+      "acc_stderr,none": 0.015716169953204184
+    },
+    "blimp_passive_1": {
+      "name": "blimp_passive_1",
+      "alias": "blimp_passive_1",
+      "sample_len": 1000,
+      "acc,none": 0.857,
+      "acc_stderr,none": 0.011075814808567074
+    },
+    "blimp_passive_2": {
+      "name": "blimp_passive_2",
+      "alias": "blimp_passive_2",
+      "sample_len": 1000,
+      "acc,none": 0.853,
+      "acc_stderr,none": 0.011203415395160376
+    },
+    "blimp_principle_A_c_command": {
+      "name": "blimp_principle_A_c_command",
+      "alias": "blimp_principle_A_c_command",
+      "sample_len": 1000,
+      "acc,none": 0.489,
+      "acc_stderr,none": 0.01581547119529257
+    },
+    "blimp_principle_A_case_1": {
+      "name": "blimp_principle_A_case_1",
+      "alias": "blimp_principle_A_case_1",
+      "sample_len": 1000,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "blimp_principle_A_case_2": {
+      "name": "blimp_principle_A_case_2",
+      "alias": "blimp_principle_A_case_2",
+      "sample_len": 1000,
+      "acc,none": 0.926,
+      "acc_stderr,none": 0.0082820645127041
+    },
+    "blimp_principle_A_domain_1": {
+      "name": "blimp_principle_A_domain_1",
+      "alias": "blimp_principle_A_domain_1",
+      "sample_len": 1000,
+      "acc,none": 0.895,
+      "acc_stderr,none": 0.00969892102602496
+    },
+    "blimp_principle_A_domain_2": {
+      "name": "blimp_principle_A_domain_2",
+      "alias": "blimp_principle_A_domain_2",
+      "sample_len": 1000,
+      "acc,none": 0.693,
+      "acc_stderr,none": 0.01459328489285274
+    },
+    "blimp_principle_A_domain_3": {
+      "name": "blimp_principle_A_domain_3",
+      "alias": "blimp_principle_A_domain_3",
+      "sample_len": 1000,
+      "acc,none": 0.592,
+      "acc_stderr,none": 0.015549205052920803
+    },
+    "blimp_principle_A_reconstruction": {
+      "name": "blimp_principle_A_reconstruction",
+      "alias": "blimp_principle_A_reconstruction",
+      "sample_len": 1000,
+      "acc,none": 0.504,
+      "acc_stderr,none": 0.01581879370351084
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "name": "blimp_regular_plural_subject_verb_agreement_1",
+      "alias": "blimp_regular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.876,
+      "acc_stderr,none": 0.010427498872343873
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "name": "blimp_regular_plural_subject_verb_agreement_2",
+      "alias": "blimp_regular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.855,
+      "acc_stderr,none": 0.0111399775178902
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "name": "blimp_sentential_negation_npi_licensor_present",
+      "alias": "blimp_sentential_negation_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.985,
+      "acc_stderr,none": 0.003845749574502972
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "name": "blimp_sentential_negation_npi_scope",
+      "alias": "blimp_sentential_negation_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.321,
+      "acc_stderr,none": 0.01477082181793475
+    },
+    "blimp_sentential_subject_island": {
+      "name": "blimp_sentential_subject_island",
+      "alias": "blimp_sentential_subject_island",
+      "sample_len": 1000,
+      "acc,none": 0.51,
+      "acc_stderr,none": 0.015816135752773183
+    },
+    "blimp_superlative_quantifiers_1": {
+      "name": "blimp_superlative_quantifiers_1",
+      "alias": "blimp_superlative_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.887,
+      "acc_stderr,none": 0.010016552866696773
+    },
+    "blimp_superlative_quantifiers_2": {
+      "name": "blimp_superlative_quantifiers_2",
+      "alias": "blimp_superlative_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.815,
+      "acc_stderr,none": 0.012285191326386656
+    },
+    "blimp_tough_vs_raising_1": {
+      "name": "blimp_tough_vs_raising_1",
+      "alias": "blimp_tough_vs_raising_1",
+      "sample_len": 1000,
+      "acc,none": 0.489,
+      "acc_stderr,none": 0.01581547119529257
+    },
+    "blimp_tough_vs_raising_2": {
+      "name": "blimp_tough_vs_raising_2",
+      "alias": "blimp_tough_vs_raising_2",
+      "sample_len": 1000,
+      "acc,none": 0.892,
+      "acc_stderr,none": 0.009820001651345708
+    },
+    "blimp_transitive": {
+      "name": "blimp_transitive",
+      "alias": "blimp_transitive",
+      "sample_len": 1000,
+      "acc,none": 0.816,
+      "acc_stderr,none": 0.01225945734093864
+    },
+    "blimp_wh_island": {
+      "name": "blimp_wh_island",
+      "alias": "blimp_wh_island",
+      "sample_len": 1000,
+      "acc,none": 0.844,
+      "acc_stderr,none": 0.011480235006122295
+    },
+    "blimp_wh_questions_object_gap": {
+      "name": "blimp_wh_questions_object_gap",
+      "alias": "blimp_wh_questions_object_gap",
+      "sample_len": 1000,
+      "acc,none": 0.569,
+      "acc_stderr,none": 0.015667944488173442
+    },
+    "blimp_wh_questions_subject_gap": {
+      "name": "blimp_wh_questions_subject_gap",
+      "alias": "blimp_wh_questions_subject_gap",
+      "sample_len": 1000,
+      "acc,none": 0.853,
+      "acc_stderr,none": 0.011203415395160376
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "name": "blimp_wh_questions_subject_gap_long_distance",
+      "alias": "blimp_wh_questions_subject_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.802,
+      "acc_stderr,none": 0.012607733934175283
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "name": "blimp_wh_vs_that_no_gap",
+      "alias": "blimp_wh_vs_that_no_gap",
+      "sample_len": 1000,
+      "acc,none": 0.953,
+      "acc_stderr,none": 0.006695956678163013
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "name": "blimp_wh_vs_that_no_gap_long_distance",
+      "alias": "blimp_wh_vs_that_no_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.962,
+      "acc_stderr,none": 0.0060491811505849775
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "name": "blimp_wh_vs_that_with_gap",
+      "alias": "blimp_wh_vs_that_with_gap",
+      "sample_len": 1000,
+      "acc,none": 0.47,
+      "acc_stderr,none": 0.015790799515836725
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "name": "blimp_wh_vs_that_with_gap_long_distance",
+      "alias": "blimp_wh_vs_that_with_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.238,
+      "acc_stderr,none": 0.013473586661967093
+    },
+    "blimp": {
+      "alias": "blimp",
+      "name": "blimp",
+      "sample_len": 67000,
+      "acc,none": 0.7445970149253731,
+      "acc_stderr,none": 0.0015076692481351098,
+      "sample_count": {
+        "acc,none": 67000
+      }
+    }
+  }
+}

m03/config.json

@@ -0,0 +1,49 @@
+{
+  "model_id": "m03",
+  "run_name": "m03-mqa-learned-dense-adamw",
+  "data_dir": "data/fineweb-edu",
+  "vocab_size": 50257,
+  "block_size": 1024,
+  "d_model": 512,
+  "num_layers": 12,
+  "num_heads": 8,
+  "ffn_mult": 4,
+  "dropout": 0.0,
+  "bias": false,
+  "norm_type": "rmsnorm",
+  "tie_embeddings": true,
+  "attention_type": "mqa",
+  "gqa_num_kv_heads": 2,
+  "mla_latent_dim": 64,
+  "pos_type": "learned",
+  "rope_base": 10000,
+  "ffn_type": "dense",
+  "moe_num_experts": 8,
+  "moe_top_k": 2,
+  "optimizer": "adamw",
+  "lr": 0.0006,
+  "min_lr": 6e-05,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "muon_lr": 0.02,
+  "muon_momentum": 0.95,
+  "training_tokens": 1000000000,
+  "warmup_tokens": 6000000,
+  "lr_schedule": "cosine",
+  "batch_size": 4,
+  "grad_accum_steps": 16,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "seed": 1337,
+  "eval_interval": 250,
+  "eval_iters": 100,
+  "log_interval": 10,
+  "checkpoint_interval": 1000,
+  "results_dir": "results",
+  "mlflow_experiment": "slm-ablation",
+  "mlflow_tracking_uri": "sqlite:///mlflow.db",
+  "mlflow_artifact_location": "./mlartifacts"
+}

m03/eval_results.json

@@ -0,0 +1,212 @@
+{
+  "model_id": "m03",
+  "checkpoint": "results/m03/final.pt",
+  "trained_step": 15258,
+  "total_params": 58512384,
+  "active_params": 58512384,
+  "flops_per_forward": 117024768.0,
+  "val_loss": 3.5667272806167603,
+  "val_perplexity": 35.400547305117705,
+  "bits_per_token": 5.145699759949179,
+  "ppl_tokens_evaluated": 655360,
+  "prefill_tok_per_sec": 187037.5785525707,
+  "inference_tok_per_sec": 280.96421214810846,
+  "inference_tok_per_sec_kvcache": 277.8238173477229,
+  "kvcache_speedup": 0.9888227942755567,
+  "gen_tokens": 128,
+  "peak_infer_vram_gb": 0.361,
+  "peak_train_vram_gb": 3.972,
+  "closeness_answer_ppl": 28.406,
+  "closeness_hit_rate": 0.35,
+  "closeness_token_f1": 0.031,
+  "closeness_n": 20,
+  "closeness_items": [
+    {
+      "prompt": "The capital of France is",
+      "answer": " Paris",
+      "generated": " the French capital of the French",
+      "answer_nll": 3.6381,
+      "answer_ppl": 38.02,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The capital of Japan is",
+      "answer": " Tokyo",
+      "generated": " the capital of the United States",
+      "answer_nll": 3.3022,
+      "answer_ppl": 27.173,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The largest planet in the solar system is",
+      "answer": " Jupiter",
+      "generated": " the Sun, which is the",
+      "answer_nll": 2.7745,
+      "answer_ppl": 16.03,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Water is made of hydrogen and",
+      "answer": " oxygen",
+      "generated": " oxygen.\nThe hydrogen in",
+      "answer_nll": 1.0093,
+      "answer_ppl": 2.744,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "answer": " cold",
+      "generated": " the case.\nThe following",
+      "answer_nll": 5.3098,
+      "answer_ppl": 202.3,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of up is",
+      "answer": " down",
+      "generated": " the up.\nThe up",
+      "answer_nll": 6.1907,
+      "answer_ppl": 488.17,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Two plus two equals",
+      "answer": " four",
+      "generated": " two equals two equals two equals",
+      "answer_nll": 3.0375,
+      "answer_ppl": 20.854,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The sun rises in the",
+      "answer": " east",
+      "generated": " sky, and the moon rises",
+      "answer_nll": 3.8344,
+      "answer_ppl": 46.267,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The first president of the United States was George",
+      "answer": " Washington",
+      "generated": " Washington, who was the first",
+      "answer_nll": 1.1363,
+      "answer_ppl": 3.115,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The speed of light is approximately 300,000 kilometers per",
+      "answer": " second",
+      "generated": " second.\nThe light is",
+      "answer_nll": 0.1141,
+      "answer_ppl": 1.121,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "A triangle has three",
+      "answer": " sides",
+      "generated": " sides, and the sides are",
+      "answer_nll": 1.0904,
+      "answer_ppl": 2.976,
+      "hit": true,
+      "token_f1": 0.333
+    },
+    {
+      "prompt": "The chemical symbol for gold is",
+      "answer": " Au",
+      "generated": " the gold.\nThe symbol",
+      "answer_nll": 8.8224,
+      "answer_ppl": 6784.315,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The freezing point of water in Celsius is",
+      "answer": " zero",
+      "generated": " the freezing point of water.",
+      "answer_nll": 5.5041,
+      "answer_ppl": 245.686,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The author of Romeo and Juliet is William",
+      "answer": " Shakespeare",
+      "generated": " Shakespeare\u2019s most famous",
+      "answer_nll": 0.1638,
+      "answer_ppl": 1.178,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The Earth revolves around the",
+      "answer": " Sun",
+      "generated": " sun, and the moon,",
+      "answer_nll": 1.4066,
+      "answer_ppl": 4.082,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The plural of mouse is",
+      "answer": " mice",
+      "generated": " the mouse.\nThe mouse",
+      "answer_nll": 8.5882,
+      "answer_ppl": 5367.751,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Photosynthesis occurs in the",
+      "answer": " leaves",
+      "generated": " soil and the soil is not",
+      "answer_nll": 3.3132,
+      "answer_ppl": 27.473,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The human heart pumps",
+      "answer": " blood",
+      "generated": " blood to the brain and the",
+      "answer_nll": 1.8433,
+      "answer_ppl": 6.317,
+      "hit": true,
+      "token_f1": 0.286
+    },
+    {
+      "prompt": "The currency used in the United States is the",
+      "answer": " dollar",
+      "generated": " United States of America.\n",
+      "answer_nll": 3.0758,
+      "answer_ppl": 21.667,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The tallest animal in the world is the",
+      "answer": " giraffe",
+      "generated": " giant panda, which is",
+      "answer_nll": 2.7774,
+      "answer_ppl": 16.076,
+      "hit": false,
+      "token_f1": 0.0
+    }
+  ],
+  "generation_samples": {
+    "The most important thing about sleep is": "The most important thing about sleep is to do a few things. You can do some rest and sleep by doing things like eating a lot of sugary foods.\n- Take a few minutes to get the sleep you want and try to wake up. Maybe you need to go for a walk.\n- Work on a healthy bedtime",
+    "Scientists have discovered that": "Scientists have discovered that a protein in the inner bark of a tree provides energy to the soil and water.\nThe scientists found that the chemical compound of alkaloids, the minerals that are used to create water and the mineral is one of the earliest forms of energy.\nThere are many things that scientists think this type",
+    "In the history of mathematics,": "In the history of mathematics, the term arithmetic is used, and it has become one of the most popular fields of study. It is widely used to refer to mathematical methods that have not been widely used before, and for the most part, it has been used in other branches of mathematics, such as arithmetic, or the study of",
+    "Python code for prime number": "Python code for prime number 32, decimal number 52, and the string 's3' for integers and numbers in the order given by the operator (see the first and last bytes of the string).\nA prime number is an array of numbers that are declared on a byte of the prime numbers. Like a prime number,",
+    "The best way to learn a new language is": "The best way to learn a new language is to learn about the world around you. It is especially important to learn new words. If you have noticed a few words, you should have a strong grasp of how to use the language and the way to learn a new language. Learning the language will help you to learn various language skills with a boost"
+  }
+}

m03/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a33e9f0e101ce7b46604d249cd4a663a6d64210c0458f507241abb8aef50ed47
+size 234059112