Add m27

m27/benchmark_results.json

@@ -0,0 +1,540 @@
+{
+  "model_id": "m27",
+  "checkpoint": "results/m27/final.pt",
+  "trained_step": 15258,
+  "num_fewshot": 0,
+  "limit": null,
+  "results": {
+    "lambada_openai": {
+      "name": "lambada_openai",
+      "alias": "lambada_openai",
+      "sample_len": 5153,
+      "perplexity,none": 570.3424875824041,
+      "perplexity_stderr,none": 30.24159359562431,
+      "acc,none": 0.16553464001552493,
+      "acc_stderr,none": 0.005177983637644952
+    },
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.27365066719776937,
+      "acc_stderr,none": 0.0044492062959226135,
+      "acc_norm,none": 0.2861979685321649,
+      "acc_norm_stderr,none": 0.004510593395290067
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.6055495103373232,
+      "acc_stderr,none": 0.011402931101558359,
+      "acc_norm,none": 0.5963003264417845,
+      "acc_norm_stderr,none": 0.011447407541749126
+    },
+    "sciq": {
+      "name": "sciq",
+      "alias": "sciq",
+      "sample_len": 1000,
+      "acc,none": 0.695,
+      "acc_stderr,none": 0.014566646394664325,
+      "acc_norm,none": 0.608,
+      "acc_norm_stderr,none": 0.015445859463771338
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.44654882154882153,
+      "acc_stderr,none": 0.010200990076245286,
+      "acc_norm,none": 0.4074074074074074,
+      "acc_norm_stderr,none": 0.010082326627832896
+    },
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.5169692186266772,
+      "acc_stderr,none": 0.014044390401613116
+    },
+    "blimp_adjunct_island": {
+      "name": "blimp_adjunct_island",
+      "alias": "blimp_adjunct_island",
+      "sample_len": 1000,
+      "acc,none": 0.812,
+      "acc_stderr,none": 0.012361586015103758
+    },
+    "blimp_anaphor_gender_agreement": {
+      "name": "blimp_anaphor_gender_agreement",
+      "alias": "blimp_anaphor_gender_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.892,
+      "acc_stderr,none": 0.009820001651345708
+    },
+    "blimp_anaphor_number_agreement": {
+      "name": "blimp_anaphor_number_agreement",
+      "alias": "blimp_anaphor_number_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.983,
+      "acc_stderr,none": 0.004089954489689063
+    },
+    "blimp_animate_subject_passive": {
+      "name": "blimp_animate_subject_passive",
+      "alias": "blimp_animate_subject_passive",
+      "sample_len": 1000,
+      "acc,none": 0.735,
+      "acc_stderr,none": 0.013963164754810064
+    },
+    "blimp_animate_subject_trans": {
+      "name": "blimp_animate_subject_trans",
+      "alias": "blimp_animate_subject_trans",
+      "sample_len": 1000,
+      "acc,none": 0.891,
+      "acc_stderr,none": 0.009859828407037124
+    },
+    "blimp_causative": {
+      "name": "blimp_causative",
+      "alias": "blimp_causative",
+      "sample_len": 1000,
+      "acc,none": 0.713,
+      "acc_stderr,none": 0.014312087053810001
+    },
+    "blimp_complex_NP_island": {
+      "name": "blimp_complex_NP_island",
+      "alias": "blimp_complex_NP_island",
+      "sample_len": 1000,
+      "acc,none": 0.553,
+      "acc_stderr,none": 0.015730176046009098
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "sample_len": 1000,
+      "acc,none": 0.515,
+      "acc_stderr,none": 0.01581217964181488
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "name": "blimp_coordinate_structure_constraint_object_extraction",
+      "alias": "blimp_coordinate_structure_constraint_object_extraction",
+      "sample_len": 1000,
+      "acc,none": 0.742,
+      "acc_stderr,none": 0.013842963108656601
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "name": "blimp_determiner_noun_agreement_1",
+      "alias": "blimp_determiner_noun_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.962,
+      "acc_stderr,none": 0.0060491811505849775
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "name": "blimp_determiner_noun_agreement_2",
+      "alias": "blimp_determiner_noun_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.944,
+      "acc_stderr,none": 0.007274401481697097
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.889,
+      "acc_stderr,none": 0.009938701010583716
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.919,
+      "acc_stderr,none": 0.008632121032139986
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_2",
+      "sample_len": 1000,
+      "acc,none": 0.896,
+      "acc_stderr,none": 0.009658016218524242
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.855,
+      "acc_stderr,none": 0.0111399775178902
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.89,
+      "acc_stderr,none": 0.00989939381972442
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "name": "blimp_determiner_noun_agreement_with_adjective_1",
+      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
+      "sample_len": 1000,
+      "acc,none": 0.9,
+      "acc_stderr,none": 0.00949157995752499
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "name": "blimp_distractor_agreement_relational_noun",
+      "alias": "blimp_distractor_agreement_relational_noun",
+      "sample_len": 1000,
+      "acc,none": 0.793,
+      "acc_stderr,none": 0.012818553557844009
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "name": "blimp_distractor_agreement_relative_clause",
+      "alias": "blimp_distractor_agreement_relative_clause",
+      "sample_len": 1000,
+      "acc,none": 0.602,
+      "acc_stderr,none": 0.015486634102859016
+    },
+    "blimp_drop_argument": {
+      "name": "blimp_drop_argument",
+      "alias": "blimp_drop_argument",
+      "sample_len": 1000,
+      "acc,none": 0.78,
+      "acc_stderr,none": 0.013106173040661862
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "name": "blimp_ellipsis_n_bar_1",
+      "alias": "blimp_ellipsis_n_bar_1",
+      "sample_len": 1000,
+      "acc,none": 0.73,
+      "acc_stderr,none": 0.01404625563263382
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "name": "blimp_ellipsis_n_bar_2",
+      "alias": "blimp_ellipsis_n_bar_2",
+      "sample_len": 1000,
+      "acc,none": 0.854,
+      "acc_stderr,none": 0.011171786285496551
+    },
+    "blimp_existential_there_object_raising": {
+      "name": "blimp_existential_there_object_raising",
+      "alias": "blimp_existential_there_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.769,
+      "acc_stderr,none": 0.013334797216936478
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "name": "blimp_existential_there_quantifiers_1",
+      "alias": "blimp_existential_there_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.972,
+      "acc_stderr,none": 0.005219506034410081
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "name": "blimp_existential_there_quantifiers_2",
+      "alias": "blimp_existential_there_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.417,
+      "acc_stderr,none": 0.015599819048769583
+    },
+    "blimp_existential_there_subject_raising": {
+      "name": "blimp_existential_there_subject_raising",
+      "alias": "blimp_existential_there_subject_raising",
+      "sample_len": 1000,
+      "acc,none": 0.857,
+      "acc_stderr,none": 0.011075814808567074
+    },
+    "blimp_expletive_it_object_raising": {
+      "name": "blimp_expletive_it_object_raising",
+      "alias": "blimp_expletive_it_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.76,
+      "acc_stderr,none": 0.013512312258920847
+    },
+    "blimp_inchoative": {
+      "name": "blimp_inchoative",
+      "alias": "blimp_inchoative",
+      "sample_len": 1000,
+      "acc,none": 0.668,
+      "acc_stderr,none": 0.014899597242811567
+    },
+    "blimp_intransitive": {
+      "name": "blimp_intransitive",
+      "alias": "blimp_intransitive",
+      "sample_len": 1000,
+      "acc,none": 0.763,
+      "acc_stderr,none": 0.013454070462577935
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "name": "blimp_irregular_past_participle_adjectives",
+      "alias": "blimp_irregular_past_participle_adjectives",
+      "sample_len": 1000,
+      "acc,none": 0.896,
+      "acc_stderr,none": 0.009658016218524242
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "name": "blimp_irregular_past_participle_verbs",
+      "alias": "blimp_irregular_past_participle_verbs",
+      "sample_len": 1000,
+      "acc,none": 0.823,
+      "acc_stderr,none": 0.012075463420375087
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_1",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.887,
+      "acc_stderr,none": 0.010016552866696773
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_2",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.879,
+      "acc_stderr,none": 0.010318210380946179
+    },
+    "blimp_left_branch_island_echo_question": {
+      "name": "blimp_left_branch_island_echo_question",
+      "alias": "blimp_left_branch_island_echo_question",
+      "sample_len": 1000,
+      "acc,none": 0.4,
+      "acc_stderr,none": 0.015499685165842571
+    },
+    "blimp_left_branch_island_simple_question": {
+      "name": "blimp_left_branch_island_simple_question",
+      "alias": "blimp_left_branch_island_simple_question",
+      "sample_len": 1000,
+      "acc,none": 0.561,
+      "acc_stderr,none": 0.015701131345400736
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "name": "blimp_matrix_question_npi_licensor_present",
+      "alias": "blimp_matrix_question_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.513,
+      "acc_stderr,none": 0.015813952101896636
+    },
+    "blimp_npi_present_1": {
+      "name": "blimp_npi_present_1",
+      "alias": "blimp_npi_present_1",
+      "sample_len": 1000,
+      "acc,none": 0.631,
+      "acc_stderr,none": 0.015266698139154782
+    },
+    "blimp_npi_present_2": {
+      "name": "blimp_npi_present_2",
+      "alias": "blimp_npi_present_2",
+      "sample_len": 1000,
+      "acc,none": 0.705,
+      "acc_stderr,none": 0.014428554438445425
+    },
+    "blimp_only_npi_licensor_present": {
+      "name": "blimp_only_npi_licensor_present",
+      "alias": "blimp_only_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.806,
+      "acc_stderr,none": 0.012510816141264347
+    },
+    "blimp_only_npi_scope": {
+      "name": "blimp_only_npi_scope",
+      "alias": "blimp_only_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.491,
+      "acc_stderr,none": 0.0158167369950053
+    },
+    "blimp_passive_1": {
+      "name": "blimp_passive_1",
+      "alias": "blimp_passive_1",
+      "sample_len": 1000,
+      "acc,none": 0.895,
+      "acc_stderr,none": 0.00969892102602496
+    },
+    "blimp_passive_2": {
+      "name": "blimp_passive_2",
+      "alias": "blimp_passive_2",
+      "sample_len": 1000,
+      "acc,none": 0.877,
+      "acc_stderr,none": 0.010391293421849803
+    },
+    "blimp_principle_A_c_command": {
+      "name": "blimp_principle_A_c_command",
+      "alias": "blimp_principle_A_c_command",
+      "sample_len": 1000,
+      "acc,none": 0.547,
+      "acc_stderr,none": 0.015749255189977687
+    },
+    "blimp_principle_A_case_1": {
+      "name": "blimp_principle_A_case_1",
+      "alias": "blimp_principle_A_case_1",
+      "sample_len": 1000,
+      "acc,none": 0.999,
+      "acc_stderr,none": 0.001
+    },
+    "blimp_principle_A_case_2": {
+      "name": "blimp_principle_A_case_2",
+      "alias": "blimp_principle_A_case_2",
+      "sample_len": 1000,
+      "acc,none": 0.923,
+      "acc_stderr,none": 0.00843458014024066
+    },
+    "blimp_principle_A_domain_1": {
+      "name": "blimp_principle_A_domain_1",
+      "alias": "blimp_principle_A_domain_1",
+      "sample_len": 1000,
+      "acc,none": 0.9,
+      "acc_stderr,none": 0.00949157995752499
+    },
+    "blimp_principle_A_domain_2": {
+      "name": "blimp_principle_A_domain_2",
+      "alias": "blimp_principle_A_domain_2",
+      "sample_len": 1000,
+      "acc,none": 0.809,
+      "acc_stderr,none": 0.012436787112179482
+    },
+    "blimp_principle_A_domain_3": {
+      "name": "blimp_principle_A_domain_3",
+      "alias": "blimp_principle_A_domain_3",
+      "sample_len": 1000,
+      "acc,none": 0.658,
+      "acc_stderr,none": 0.015008706182121804
+    },
+    "blimp_principle_A_reconstruction": {
+      "name": "blimp_principle_A_reconstruction",
+      "alias": "blimp_principle_A_reconstruction",
+      "sample_len": 1000,
+      "acc,none": 0.408,
+      "acc_stderr,none": 0.015549205052920803
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "name": "blimp_regular_plural_subject_verb_agreement_1",
+      "alias": "blimp_regular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.882,
+      "acc_stderr,none": 0.010206869264381718
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "name": "blimp_regular_plural_subject_verb_agreement_2",
+      "alias": "blimp_regular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.862,
+      "acc_stderr,none": 0.010912152632504508
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "name": "blimp_sentential_negation_npi_licensor_present",
+      "alias": "blimp_sentential_negation_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.998,
+      "acc_stderr,none": 0.0014135055705578234
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "name": "blimp_sentential_negation_npi_scope",
+      "alias": "blimp_sentential_negation_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.599,
+      "acc_stderr,none": 0.015506109745498368
+    },
+    "blimp_sentential_subject_island": {
+      "name": "blimp_sentential_subject_island",
+      "alias": "blimp_sentential_subject_island",
+      "sample_len": 1000,
+      "acc,none": 0.519,
+      "acc_stderr,none": 0.015807874268505835
+    },
+    "blimp_superlative_quantifiers_1": {
+      "name": "blimp_superlative_quantifiers_1",
+      "alias": "blimp_superlative_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.923,
+      "acc_stderr,none": 0.00843458014024066
+    },
+    "blimp_superlative_quantifiers_2": {
+      "name": "blimp_superlative_quantifiers_2",
+      "alias": "blimp_superlative_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.878,
+      "acc_stderr,none": 0.010354864712936776
+    },
+    "blimp_tough_vs_raising_1": {
+      "name": "blimp_tough_vs_raising_1",
+      "alias": "blimp_tough_vs_raising_1",
+      "sample_len": 1000,
+      "acc,none": 0.557,
+      "acc_stderr,none": 0.015716169953204184
+    },
+    "blimp_tough_vs_raising_2": {
+      "name": "blimp_tough_vs_raising_2",
+      "alias": "blimp_tough_vs_raising_2",
+      "sample_len": 1000,
+      "acc,none": 0.888,
+      "acc_stderr,none": 0.009977753031397177
+    },
+    "blimp_transitive": {
+      "name": "blimp_transitive",
+      "alias": "blimp_transitive",
+      "sample_len": 1000,
+      "acc,none": 0.815,
+      "acc_stderr,none": 0.012285191326386656
+    },
+    "blimp_wh_island": {
+      "name": "blimp_wh_island",
+      "alias": "blimp_wh_island",
+      "sample_len": 1000,
+      "acc,none": 0.767,
+      "acc_stderr,none": 0.01337497251921999
+    },
+    "blimp_wh_questions_object_gap": {
+      "name": "blimp_wh_questions_object_gap",
+      "alias": "blimp_wh_questions_object_gap",
+      "sample_len": 1000,
+      "acc,none": 0.708,
+      "acc_stderr,none": 0.014385511563477433
+    },
+    "blimp_wh_questions_subject_gap": {
+      "name": "blimp_wh_questions_subject_gap",
+      "alias": "blimp_wh_questions_subject_gap",
+      "sample_len": 1000,
+      "acc,none": 0.867,
+      "acc_stderr,none": 0.010743669132397295
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "name": "blimp_wh_questions_subject_gap_long_distance",
+      "alias": "blimp_wh_questions_subject_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.782,
+      "acc_stderr,none": 0.013063179040595235
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "name": "blimp_wh_vs_that_no_gap",
+      "alias": "blimp_wh_vs_that_no_gap",
+      "sample_len": 1000,
+      "acc,none": 0.953,
+      "acc_stderr,none": 0.006695956678163013
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "name": "blimp_wh_vs_that_no_gap_long_distance",
+      "alias": "blimp_wh_vs_that_no_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.96,
+      "acc_stderr,none": 0.006199874066337028
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "name": "blimp_wh_vs_that_with_gap",
+      "alias": "blimp_wh_vs_that_with_gap",
+      "sample_len": 1000,
+      "acc,none": 0.471,
+      "acc_stderr,none": 0.015792669451628764
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "name": "blimp_wh_vs_that_with_gap_long_distance",
+      "alias": "blimp_wh_vs_that_with_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.184,
+      "acc_stderr,none": 0.01225945734093864
+    },
+    "blimp": {
+      "alias": "blimp",
+      "name": "blimp",
+      "sample_len": 67000,
+      "acc,none": 0.7653283582089553,
+      "acc_stderr,none": 0.0014914982264402836,
+      "sample_count": {
+        "acc,none": 67000
+      }
+    }
+  }
+}

m27/config.json

@@ -0,0 +1,51 @@
+{
+  "model_id": "m27",
+  "run_name": "m27-mha-learned-moe-adamw-seed1338",
+  "data_dir": "data/fineweb-edu",
+  "vocab_size": 50257,
+  "block_size": 1024,
+  "d_model": 512,
+  "num_layers": 12,
+  "num_heads": 8,
+  "ffn_mult": 4,
+  "dropout": 0.0,
+  "bias": false,
+  "norm_type": "rmsnorm",
+  "tie_embeddings": true,
+  "attention_type": "mha",
+  "gqa_num_kv_heads": 2,
+  "mla_latent_dim": 64,
+  "pos_type": "learned",
+  "rope_base": 10000,
+  "ffn_type": "moe",
+  "moe_num_experts": 8,
+  "moe_top_k": 2,
+  "optimizer": "adamw",
+  "lr": 0.0006,
+  "min_lr": 6e-05,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "muon_lr": 0.02,
+  "muon_momentum": 0.95,
+  "training_tokens": 1000000000,
+  "warmup_tokens": 6000000,
+  "lr_schedule": "cosine",
+  "batch_size": 2,
+  "grad_accum_steps": 32,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "seed": 1338,
+  "eval_interval": 250,
+  "eval_iters": 100,
+  "log_interval": 10,
+  "checkpoint_interval": 1000,
+  "results_dir": "results",
+  "mlflow_experiment": "slm-ablation",
+  "mlflow_tracking_uri": "sqlite:///mlflow.db",
+  "mlflow_artifact_location": "./mlartifacts",
+  "moe_aux_loss_coef": 0.01,
+  "seed_group": "m02"
+}

m27/eval_results.json

@@ -0,0 +1,223 @@
+{
+  "model_id": "m27",
+  "checkpoint": "/home/akshat/codemindv2/results/m27/final.pt",
+  "trained_step": 15258,
+  "total_params": 240227328,
+  "active_params": 89232384,
+  "flops_per_forward": 178464768.0,
+  "val_loss": 3.5370666682720184,
+  "val_perplexity": 34.3659644225298,
+  "bits_per_token": 5.10290854160969,
+  "ppl_tokens_evaluated": 655360,
+  "prefill_tok_per_sec": 79516.50626434464,
+  "inference_tok_per_sec": 46.00478444832356,
+  "inference_tok_per_sec_kvcache": null,
+  "kvcache_speedup": null,
+  "gen_tokens": 128,
+  "peak_infer_vram_gb": 1.451,
+  "peak_train_vram_gb": 3.604,
+  "closeness_answer_ppl": 20.233,
+  "closeness_hit_rate": 0.55,
+  "closeness_token_f1": 0.053,
+  "closeness_n": 20,
+  "closeness_items": [
+    {
+      "prompt": "The capital of France is",
+      "answer": " Paris",
+      "generated": " Paris.\nThe capital of",
+      "answer_nll": 1.1664,
+      "answer_ppl": 3.211,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The capital of Japan is",
+      "answer": " Tokyo",
+      "generated": " Tokyo.\nThe capital of",
+      "answer_nll": 1.4125,
+      "answer_ppl": 4.106,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The largest planet in the solar system is",
+      "answer": " Jupiter",
+      "generated": " Mercury, which is about 1",
+      "answer_nll": 3.1623,
+      "answer_ppl": 23.625,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Water is made of hydrogen and",
+      "answer": " oxygen",
+      "generated": " oxygen.\n- The hydrogen",
+      "answer_nll": 0.8734,
+      "answer_ppl": 2.395,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "answer": " cold",
+      "generated": " the heat of the sun.",
+      "answer_nll": 3.3773,
+      "answer_ppl": 29.291,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of up is",
+      "answer": " down",
+      "generated": " the fact that the two are",
+      "answer_nll": 7.6189,
+      "answer_ppl": 2036.362,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Two plus two equals",
+      "answer": " four",
+      "generated": " two.\nThe first two",
+      "answer_nll": 3.9553,
+      "answer_ppl": 52.211,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The sun rises in the",
+      "answer": " east",
+      "generated": " east, and the sun is",
+      "answer_nll": 1.7612,
+      "answer_ppl": 5.82,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The first president of the United States was George",
+      "answer": " Washington",
+      "generated": " Washington.\nThe first president",
+      "answer_nll": 0.4384,
+      "answer_ppl": 1.55,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The speed of light is approximately 300,000 kilometers per",
+      "answer": " second",
+      "generated": " second.\nThe speed of",
+      "answer_nll": 0.0309,
+      "answer_ppl": 1.031,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "A triangle has three",
+      "answer": " sides",
+      "generated": " sides and four sides.\n",
+      "answer_nll": 1.534,
+      "answer_ppl": 4.637,
+      "hit": true,
+      "token_f1": 0.4
+    },
+    {
+      "prompt": "The chemical symbol for gold is",
+      "answer": " Au",
+      "generated": " gold.\nThe symbol for",
+      "answer_nll": 8.3869,
+      "answer_ppl": 4389.135,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The freezing point of water in Celsius is",
+      "answer": " zero",
+      "generated": " 0.5 degrees Celsius.",
+      "answer_nll": 6.6989,
+      "answer_ppl": 811.538,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The author of Romeo and Juliet is William",
+      "answer": " Shakespeare",
+      "generated": " Shakespeare.\nThe book is",
+      "answer_nll": 0.9888,
+      "answer_ppl": 2.688,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The Earth revolves around the",
+      "answer": " Sun",
+      "generated": " Sun, and the Sun revolves",
+      "answer_nll": 0.5558,
+      "answer_ppl": 1.743,
+      "hit": true,
+      "token_f1": 0.333
+    },
+    {
+      "prompt": "The plural of mouse is",
+      "answer": " mice",
+      "generated": " the plural of mouse is the",
+      "answer_nll": 7.4291,
+      "answer_ppl": 1684.289,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Photosynthesis occurs in the",
+      "answer": " leaves",
+      "generated": " form of a single cell.",
+      "answer_nll": 3.8019,
+      "answer_ppl": 44.787,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The human heart pumps",
+      "answer": " blood",
+      "generated": " blood to the lungs, where",
+      "answer_nll": 1.0128,
+      "answer_ppl": 2.753,
+      "hit": true,
+      "token_f1": 0.333
+    },
+    {
+      "prompt": "The currency used in the United States is the",
+      "answer": " dollar",
+      "generated": " U.S. dollar.",
+      "answer_nll": 3.3461,
+      "answer_ppl": 28.392,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The tallest animal in the world is the",
+      "answer": " giraffe",
+      "generated": " giant panda.\nThe",
+      "answer_nll": 2.5951,
+      "answer_ppl": 13.399,
+      "hit": false,
+      "token_f1": 0.0
+    }
+  ],
+  "moe/aux_loss": 0.1201492833904922,
+  "moe/routing_entropy": 0.9966633478800457,
+  "moe/max_load_frac": 0.13975499471028646,
+  "moe/expert_0_prob": 0.1208117138594389,
+  "moe/expert_1_prob": 0.12582085505127907,
+  "moe/expert_2_prob": 0.13036262318491937,
+  "moe/expert_3_prob": 0.12241238094866276,
+  "moe/expert_4_prob": 0.1250034064054489,
+  "moe/expert_5_prob": 0.12479039952158928,
+  "moe/expert_6_prob": 0.12319008596241474,
+  "moe/expert_7_prob": 0.12760853469371797,
+  "generation_samples": {
+    "The most important thing about sleep is": "The most important thing about sleep is that it helps you sleep better.\nSleep is the most crucial element in our bodies. Research shows that sleep improves your immune system. When we wake up, our sleep gets easier because we are sleep deprived.\nSleep has a great influence on our emotions, especially our thoughts. It improves our mood",
+    "Scientists have discovered that": "Scientists have discovered that when their muscles contract, protein-based proteins and peptides, which are critical to the development of the heart, are released when they\u2019re ready to move forward. In the case of a heart attack, the body produces a hormone called vasopressin, which is released and signals the",
+    "In the history of mathematics,": "In the history of mathematics, we have very much more information about how our brain works than most people have ever heard. For example, we do not quite understand what the brain does: We do not have much to learn about the brain itself. And we do not know how, or even if, this information is useful.\n",
+    "Python code for prime number": "Python code for prime number, and also for the sequence of steps, as well as the pattern of squares, as in the example below.\n- In the example above, only the first number is used in the sequence.\n- If the last number is the first of the numbers in the sequence of the steps, the",
+    "The best way to learn a new language is": "The best way to learn a new language is to read and write one word at a time. Then, try to figure out the sentence that best fits your needs and wants to help you with reading and writing.\nIf you are interested in learning more, then check out our new language learning learning guide, T2 English Language Learning Guide.\n"
+  }
+}

m27/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a58d4815277b7f1f408570b4a9f9e9c8b90fbda1d7dc684244ca878f52de5589
+size 960938984