Upload folder using huggingface_hub

.gitattributes

@@ -84,3 +84,5 @@ llmtf_eval_k5_bs8/darumeru_ruOpenBookQA.jsonl filter=lfs diff=lfs merge=lfs -tex
 llmtf_eval_k5_bs8/darumeru_ruTiE.jsonl filter=lfs diff=lfs merge=lfs -text
 llmtf_eval_k5_bs8/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
 llmtf_eval_k5_bs8/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5_bs8/darumeru_USE.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5_bs8/russiannlp_rucola_custom.jsonl filter=lfs diff=lfs merge=lfs -text

llmtf_eval_k5_bs8/darumeru_USE_params.jsonl

@@ -0,0 +1,57 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "NousResearch/Meta-Llama-3-8B-Instruct",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                128009
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval_k5_bs8/darumeru_USE_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/USE",
+    "results": {
+        "grade_norm": 0.18725490196078434
+    },
+    "leaderboard_result": 0.18725490196078434
+}

llmtf_eval_k5_bs8/darumeru_ruMMLU.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21fd0a778fd73c6ca1a894afced27d7e372e1ca1463aaabad6bd7eb6120223f2
-size 71218197
+oid sha256:93da8b535385d1a614578802c57d0c2f7e3bb75760324e06095cacbbbe551288
+size 95757423

llmtf_eval_k5_bs8/darumeru_ruMMLU_params.jsonl

@@ -0,0 +1,57 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "NousResearch/Meta-Llama-3-8B-Instruct",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                128009
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k5_bs8/darumeru_ruMMLU_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/ruMMLU",
+    "results": {
+        "acc": 0.5138182180983737
+    },
+    "leaderboard_result": 0.5138182180983737
+}

llmtf_eval_k5_bs8/evaluation_log.txt

@@ -107,3 +107,167 @@ INFO: 2024-07-13 14:46:14,658: llmtf.base.evaluator: Ended eval
 INFO: 2024-07-13 14:46:14,684: llmtf.base.evaluator: 
 mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree
 0.684	0.247	0.397	0.432	0.850	0.481	0.544	0.973	0.977	1.000	0.998	0.768	0.351	0.875
+INFO: 2024-07-13 14:48:58,982: llmtf.base.darumeru/USE: Processing Dataset: 204.37s
+INFO: 2024-07-13 14:48:58,999: llmtf.base.darumeru/USE: Results for darumeru/USE:
+INFO: 2024-07-13 14:48:59,004: llmtf.base.darumeru/USE: {'grade_norm': 0.18725490196078434}
+INFO: 2024-07-13 14:48:59,010: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 128009]
+INFO: 2024-07-13 14:48:59,010: llmtf.base.hfmodel: Updated generation_config.stop_strings: []
+INFO: 2024-07-13 14:49:19,451: llmtf.base.russiannlp/rucola_custom: Loading Dataset: 20.44s
+INFO: 2024-07-13 14:50:14,250: llmtf.base.nlpcoreteam/enMMLU: Processing Dataset: 1036.87s
+INFO: 2024-07-13 14:50:14,255: llmtf.base.nlpcoreteam/enMMLU: Results for nlpcoreteam/enMMLU:
+INFO: 2024-07-13 14:50:14,302: llmtf.base.nlpcoreteam/enMMLU:                                        metric
+subject                                      
+abstract_algebra                     0.350000
+anatomy                              0.696296
+astronomy                            0.730263
+business_ethics                      0.700000
+clinical_knowledge                   0.754717
+college_biology                      0.812500
+college_chemistry                    0.500000
+college_computer_science             0.590000
+college_mathematics                  0.330000
+college_medicine                     0.670520
+college_physics                      0.470588
+computer_security                    0.780000
+conceptual_physics                   0.570213
+econometrics                         0.561404
+electrical_engineering               0.634483
+elementary_mathematics               0.439153
+formal_logic                         0.507937
+global_facts                         0.430000
+high_school_biology                  0.800000
+high_school_chemistry                0.517241
+high_school_computer_science         0.760000
+high_school_european_history         0.787879
+high_school_geography                0.843434
+high_school_government_and_politics  0.922280
+high_school_macroeconomics           0.671795
+high_school_mathematics              0.381481
+high_school_microeconomics           0.764706
+high_school_physics                  0.417219
+high_school_psychology               0.847706
+high_school_statistics               0.537037
+high_school_us_history               0.833333
+high_school_world_history            0.835443
+human_aging                          0.730942
+human_sexuality                      0.801527
+international_law                    0.818182
+jurisprudence                        0.759259
+logical_fallacies                    0.766871
+machine_learning                     0.544643
+management                           0.825243
+marketing                            0.901709
+medical_genetics                     0.830000
+miscellaneous                        0.842912
+moral_disputes                       0.751445
+moral_scenarios                      0.497207
+nutrition                            0.754902
+philosophy                           0.720257
+prehistory                           0.753086
+professional_accounting              0.556738
+professional_law                     0.483051
+professional_medicine                0.742647
+professional_psychology              0.717320
+public_relations                     0.690909
+security_studies                     0.722449
+sociology                            0.840796
+us_foreign_policy                    0.840000
+virology                             0.512048
+world_religions                      0.818713
+INFO: 2024-07-13 14:50:14,310: llmtf.base.nlpcoreteam/enMMLU:                                    metric
+subject                                  
+STEM                             0.564712
+humanities                       0.717897
+other (business, health, misc.)  0.710620
+social sciences                  0.768694
+INFO: 2024-07-13 14:50:14,318: llmtf.base.nlpcoreteam/enMMLU: {'acc': 0.6904807286717012}
+INFO: 2024-07-13 14:50:14,385: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-13 14:50:14,399: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU
+0.651	0.247	0.397	0.432	0.850	0.481	0.544	0.187	0.973	0.977	1.000	0.998	0.768	0.351	0.875	0.690
+INFO: 2024-07-13 14:51:55,784: llmtf.base.darumeru/ruMMLU: Processing Dataset: 1262.30s
+INFO: 2024-07-13 14:51:55,788: llmtf.base.darumeru/ruMMLU: Results for darumeru/ruMMLU:
+INFO: 2024-07-13 14:51:55,799: llmtf.base.darumeru/ruMMLU: {'acc': 0.5138182180983737}
+INFO: 2024-07-13 14:51:55,888: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-13 14:51:55,906: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU
+0.643	0.247	0.397	0.432	0.850	0.481	0.544	0.187	0.973	0.977	1.000	0.998	0.514	0.768	0.351	0.875	0.690
+INFO: 2024-07-13 14:52:18,001: llmtf.base.russiannlp/rucola_custom: Processing Dataset: 178.55s
+INFO: 2024-07-13 14:52:18,002: llmtf.base.russiannlp/rucola_custom: Results for russiannlp/rucola_custom:
+INFO: 2024-07-13 14:52:18,035: llmtf.base.russiannlp/rucola_custom: {'acc': 0.7115177610333692, 'mcc': 0.3362227509262135}
+INFO: 2024-07-13 14:52:18,046: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-13 14:52:18,077: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	russiannlp/rucola_custom
+0.636	0.247	0.397	0.432	0.850	0.481	0.544	0.187	0.973	0.977	1.000	0.998	0.514	0.768	0.351	0.875	0.690	0.524
+INFO: 2024-07-13 14:59:07,852: llmtf.base.nlpcoreteam/ruMMLU: Processing Dataset: 1522.57s
+INFO: 2024-07-13 14:59:07,871: llmtf.base.nlpcoreteam/ruMMLU: Results for nlpcoreteam/ruMMLU:
+INFO: 2024-07-13 14:59:07,917: llmtf.base.nlpcoreteam/ruMMLU:                                        metric
+subject                                      
+abstract_algebra                     0.330000
+anatomy                              0.511111
+astronomy                            0.651316
+business_ethics                      0.680000
+clinical_knowledge                   0.588679
+college_biology                      0.534722
+college_chemistry                    0.480000
+college_computer_science             0.520000
+college_mathematics                  0.350000
+college_medicine                     0.549133
+college_physics                      0.352941
+computer_security                    0.720000
+conceptual_physics                   0.540426
+econometrics                         0.438596
+electrical_engineering               0.572414
+elementary_mathematics               0.417989
+formal_logic                         0.396825
+global_facts                         0.370000
+high_school_biology                  0.664516
+high_school_chemistry                0.394089
+high_school_computer_science         0.690000
+high_school_european_history         0.763636
+high_school_geography                0.666667
+high_school_government_and_politics  0.647668
+high_school_macroeconomics           0.553846
+high_school_mathematics              0.348148
+high_school_microeconomics           0.546218
+high_school_physics                  0.410596
+high_school_psychology               0.682569
+high_school_statistics               0.449074
+high_school_us_history               0.691176
+high_school_world_history            0.734177
+human_aging                          0.538117
+human_sexuality                      0.641221
+international_law                    0.743802
+jurisprudence                        0.657407
+logical_fallacies                    0.558282
+machine_learning                     0.401786
+management                           0.689320
+marketing                            0.730769
+medical_genetics                     0.670000
+miscellaneous                        0.650064
+moral_disputes                       0.630058
+moral_scenarios                      0.382123
+nutrition                            0.604575
+philosophy                           0.614148
+prehistory                           0.574074
+professional_accounting              0.397163
+professional_law                     0.397001
+professional_medicine                0.514706
+professional_psychology              0.514706
+public_relations                     0.609091
+security_studies                     0.657143
+sociology                            0.676617
+us_foreign_policy                    0.740000
+virology                             0.457831
+world_religions                      0.695906
+INFO: 2024-07-13 14:59:07,924: llmtf.base.nlpcoreteam/ruMMLU:                                    metric
+subject                                  
+STEM                             0.490445
+humanities                       0.602971
+other (business, health, misc.)  0.567962
+social sciences                  0.614529
+INFO: 2024-07-13 14:59:07,947: llmtf.base.nlpcoreteam/ruMMLU: {'acc': 0.5689766403256171}
+INFO: 2024-07-13 14:59:08,029: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-13 14:59:08,049: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.632	0.247	0.397	0.432	0.850	0.481	0.544	0.187	0.973	0.977	1.000	0.998	0.514	0.768	0.351	0.875	0.690	0.569	0.524

llmtf_eval_k5_bs8/evaluation_results.txt

@@ -1,2 +1,2 @@
-mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree
-0.684	0.247	0.397	0.432	0.850	0.481	0.544	0.973	0.977	1.000	0.998	0.768	0.351	0.875
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.632	0.247	0.397	0.432	0.850	0.481	0.544	0.187	0.973	0.977	1.000	0.998	0.514	0.768	0.351	0.875	0.690	0.569	0.524

llmtf_eval_k5_bs8/nlpcoreteam_enMMLU.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6d6e7452aa5fb69fc90ce1043ec813596a88e68d975e02617b4a9ef072a353d
-size 67044933
+oid sha256:8095dc31486a7450b06dfe16a52353d758b19897f0f9e135d258aefa5f4544a2
+size 82136115

llmtf_eval_k5_bs8/nlpcoreteam_enMMLU_params.jsonl

@@ -0,0 +1,57 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "NousResearch/Meta-Llama-3-8B-Instruct",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                128009
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k5_bs8/nlpcoreteam_enMMLU_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "nlpcoreteam/enMMLU",
+    "results": {
+        "acc": 0.6904807286717012
+    },
+    "leaderboard_result": 0.6904807286717012
+}

llmtf_eval_k5_bs8/nlpcoreteam_ruMMLU.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d9f3227af64e84961c6d5415223c58db1e568796143a7248fb781ba986b53f2
-size 62853990
+oid sha256:b0c6923c896671e3781de03ca9d473dd1d46bfb78adb8ae77032d01d297855ea
+size 119771974

llmtf_eval_k5_bs8/nlpcoreteam_ruMMLU_params.jsonl

@@ -0,0 +1,57 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "NousResearch/Meta-Llama-3-8B-Instruct",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                128009
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k5_bs8/nlpcoreteam_ruMMLU_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "nlpcoreteam/ruMMLU",
+    "results": {
+        "acc": 0.5689766403256171
+    },
+    "leaderboard_result": 0.5689766403256171
+}

llmtf_eval_k5_bs8/russiannlp_rucola_custom.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4be29752e06af3c15c296cce6829f0d26c34edc0764325aa8a4338974c6862bb
+size 13288973

llmtf_eval_k5_bs8/russiannlp_rucola_custom_params.jsonl

@@ -0,0 +1,57 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "NousResearch/Meta-Llama-3-8B-Instruct",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                128009
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 1,
+            "pad_token_id": 128001,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k5_bs8/russiannlp_rucola_custom_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "russiannlp/rucola_custom",
+    "results": {
+        "acc": 0.7115177610333692,
+        "mcc": 0.3362227509262135
+    },
+    "leaderboard_result": 0.5238702559797913
+}