Add files using upload-large-folder tool

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_easy.yaml

@@ -0,0 +1,5 @@
+dataset_name: icd10proc_easy
+include: _default_template_yaml
+tag: med_concepts_qa_icd10proc_tasks
+task: med_concepts_qa_icd10proc_easy
+task_alias: icd10proc_easy

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/medqa/preprocess_medqa.py

@@ -0,0 +1,13 @@
+def doc_to_text(doc) -> str:
+    option_choices = {
+        "A": doc["ending0"],
+        "B": doc["ending1"],
+        "C": doc["ending2"],
+        "D": doc["ending3"],
+    }
+    answers = "".join((f"{k}. {v}\n") for k, v in option_choices.items())
+    return f"Question: {doc['sent1']}\n{answers}Answer:"
+
+
+def doc_to_target(doc) -> int:
+    return doc["label"]

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mela/mela_ar.yaml

@@ -0,0 +1,4 @@
+include: mela_en.yaml
+task: mela_ar
+dataset_name: ar
+training_split: null

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mela/mela_en.yaml

@@ -0,0 +1,17 @@
+task: mela_en
+dataset_path: Geralt-Targaryen/MELA
+dataset_name: en
+training_split: train
+validation_split: dev
+test_split: test
+output_type: multiple_choice
+doc_to_text: "Sentence: {{sentence}}\nDetermine whether this sentence is acceptable or unacceptable?\nA. Acceptable\nB. Unacceptable\nAnswer:"
+doc_to_choice: ["A", "B"]
+doc_to_target: "{{['B', 'A'][label]}}"
+description: "Determine whether the following sentence(s) violate certain linguistic constraints. If yes, then it is \"unacceptable\"; otherwise, \"acceptable\".\n\n"
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+metric_list:
+  - metric: mcc
+    higher_is_better: true

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mela/mela_is.yaml

@@ -0,0 +1,4 @@
+include: mela_en.yaml
+task: mela_is
+dataset_name: is
+training_split: null

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mela/mela_ru.yaml

@@ -0,0 +1,3 @@
+include: mela_en.yaml
+task: mela_ru
+dataset_name: ru

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/README.md

@@ -0,0 +1,84 @@
+# Metabench
+
+### Paper
+
+Title: `metabench` -- A Sparse Benchmark to Measure General Ability in Large Language Models
+
+Abstract: https://arxiv.org/abs/2407.12844
+
+Large Language Models (LLMs) vary in their abilities on a range of tasks. Initiatives such as the 𝙾𝚙𝚎𝚗 𝙻𝙻𝙼 𝙻𝚎𝚊𝚍𝚎𝚛𝚋𝚘𝚊𝚛𝚍 aim to quantify these differences with several large benchmarks (sets of test items to which an LLM can respond either correctly or incorrectly). However, high correlations within and between benchmark scores suggest that (1) there exists a small set of common underlying abilities that these benchmarks measure, and (2) items tap into redundant information and the benchmarks may thus be considerably compressed. We use data from $n> 5000$ LLMs to identify the most informative items of six benchmarks, ARC, GSM8K, HellaSwag, MMLU, TruthfulQA and WinoGrande (with d=28,632 items in total). From them we distill a sparse benchmark, `metabench`, that has less than $3%$ of the original size of all six benchmarks combined. This new sparse benchmark goes beyond point scores by yielding estimators of the underlying benchmark-specific abilities. We show that these estimators (1) can be used to reconstruct each original individual benchmark score with, on average, $1.5%$ root mean square error (RMSE), (2) reconstruct the original total score with $0.8%$ RMSE, and (3) have a single underlying common factor whose Spearman correlation with the total score is $r=0.93$.
+
+Homepage: https://github.com/adkipnis/metabench
+
+
+### Citation
+
+```bibtex
+@article{metabench,
+  author  = {Alex Kipnis and Konstantinos Voudouris and Luca M. Schulze Buschoff and Eric Schulz},
+  title   = {metabench - A Sparse Benchmark to Measure General Ability in Large Language Models},
+  journal = {arXiv preprint arXiv:2407.12844},
+  year    = {2024},
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+There are four groups.
+
+* `metabench` -- combines the six tasks covering the six reduced benchmarks, using the original data and transformations from the respective benchmarks, and produces an aggregated mean score. It contains a total of 858 items.
+* `metabench_permute` -- combines five tasks covering five of the reduced benchmarks, permuting the multiple choice ordering, and produces an aggregated mean score. It contains a total of 858 items. For more details, see immediately below.
+* `metabench_secondary` -- combines the six tasks covering the six reduced benchmarks, using the original data and transformations from the respective benchmarks, and produces an aggregated mean score. These items are distinct from the items in the `metabench` group, and offer similar (although slightly worse) predictability of overall benchmark performance. We include it as a secondary evaluation resource. It contains a total of 751 items.
+* `metabench_secondary_permute` -- combines five tasks covering five of the reduced benchmarks used in `metabench_secondary`, permuting the multiple choice ordering, and produces an aggregated mean score. It contains a total of 751 items. For more details, see immediately below.
+
+#### Tasks
+
+We offer four sets of tasks. The first uses the original benchmark items straight out of the box.
+
+* `metabench_arc` -- a subset of the [ARC benchmark](https://huggingface.co/datasets/allenai/ai2_arc) containing the 145 most informative items.
+* `metabench_gsm8k` -- a subset of the [GSM8K benchmark](https://huggingface.co/datasets/openai/gsm8k) containing the 237 most informative items.
+* `metabench_hellaswag` -- a subset of the [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag) benchmark containing the 93 most informative items.
+* `metabench_mmlu` -- a subset of the [MMLU benchmark](https://huggingface.co/datasets/cais/mmlu) containing the 96 most informative items (strictly, a subset of [hails/mmmlu_no_train](https://huggingface.co/datasets/hails/mmlu_no_train)).
+* `metabench_truthfulqa` -- a subset of the [TruthfulQA benchmark](https://huggingface.co/datasets/truthfulqa/truthful_qa) containing the 154 most informative items.
+* `metabench_winogrande` -- a subset of the [Winogrande benchmark](https://huggingface.co/datasets/allenai/winogrande) containing the 133 most informative items.
+
+Since the original benchmarks are open-source, there is a risk of contamination. To mitigate this risk, we also provide tasks in which the answers are shuffled. Since `GSM8K` is not a multiple-choice benchmark, it is excluded from this set.
+
+* `metabench_arc_permute` -- a subset of the [ARC benchmark](https://huggingface.co/datasets/allenai/ai2_arc) containing the 145 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark.
+* `metabench_hellaswag_permute` -- a subset of the [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag) benchmark containing the 93 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark.
+* `metabench_mmlu_permute` -- a subset of the [MMLU benchmark](https://huggingface.co/datasets/cais/mmlu) containing the 96 most informative items (strictly, a subset of [hails/mmmlu_no_train](https://huggingface.co/datasets/hails/mmlu_no_train)). The answers are randomly permuted such that the answer key is different to the original benchmark.
+* `metabench_truthfulqa_permute` -- a subset of the [TruthfulQA benchmark](https://huggingface.co/datasets/truthfulqa/truthful_qa) containing the 154 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark.
+* `metabench_winogrande_permute` -- a subset of the [Winogrande benchmark](https://huggingface.co/datasets/allenai/winogrande) containing the 133 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark.
+
+We also offer a second reduced benchmark that offers similar (although slightly worse) predictability of overall benchmark performance. We include it as a secondary evaluation resource. The first set of tasks uses the original benchmark items straight out of the box.
+
+* `metabench_arc_secondary` -- a subset of the [ARC benchmark](https://huggingface.co/datasets/allenai/ai2_arc) containing the 100 most informative items.
+* `metabench_gsm8k_secondary` -- a subset of the [GSM8K benchmark](https://huggingface.co/datasets/openai/gsm8k) containing the 249 most informative items.
+* `metabench_hellaswag_secondary` -- a subset of the [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag) benchmark containing the 58 most informative items.
+* `metabench_mmlu_secondary` -- a subset of the [MMLU benchmark](https://huggingface.co/datasets/cais/mmlu) containing the 102 most informative items (strictly, a subset of [hails/mmmlu_no_train](https://huggingface.co/datasets/hails/mmlu_no_train)).
+* `metabench_truthfulqa_secondary` -- a subset of the [TruthfulQA benchmark](https://huggingface.co/datasets/truthfulqa/truthful_qa) containing the 136 most informative items.
+* `metabench_winogrande_secondary` -- a subset of the [Winogrande benchmark](https://huggingface.co/datasets/allenai/winogrande) containing the 106 most informative items.
+
+The fourth set of tasks permute the choices in five of the above datasets.
+
+* `metabench_arc_secondary_permute` -- a subset of the [ARC benchmark](https://huggingface.co/datasets/allenai/ai2_arc) containing the 100 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark.
+* `metabench_hellaswag_secondary_permute` -- a subset of the [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag) benchmark containing the 58 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark.
+* `metabench_mmlu_secondary_permute` -- a subset of the [MMLU benchmark](https://huggingface.co/datasets/cais/mmlu) containing the 102 most informative items (strictly, a subset of [hails/mmmlu_no_train](https://huggingface.co/datasets/hails/mmlu_no_train)). The answers are randomly permuted such that the answer key is different to the original benchmark.
+* `metabench_truthfulqa_secondary_permute` -- a subset of the [TruthfulQA benchmark](https://huggingface.co/datasets/truthfulqa/truthful_qa) containing the 136 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark.
+* `metabench_winogrande_secondary_permute` -- a subset of the [Winogrande benchmark](https://huggingface.co/datasets/allenai/winogrande) containing the 106 most informative items. The answers are randomly permuted such that the answer key is different to the original benchmark.
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [X] Is the task an existing benchmark in the literature?
+  * [X] Have you referenced the original paper that introduced the task?
+  * [X] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [X] Is the "Main" variant of this task clearly denoted?
+* [X] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [X] Have you noted which, if any, published evaluation setups are matched by this variant?
+*

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_permute.yaml

@@ -0,0 +1,5 @@
+include: metabench_arc.yaml
+task: metabench_arc_permute
+process_docs: !function process_docs_permute.process_arc
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_arc_secondary_permute.yaml

@@ -0,0 +1,5 @@
+include: metabench_arc_permute.yaml
+task: metabench_arc_secondary_permute
+test_split: secondary
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_secondary.yaml

@@ -0,0 +1,5 @@
+include: metabench_hellaswag.yaml
+task: metabench_hellaswag_secondary
+test_split: secondary
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_hellaswag_secondary_permute.yaml

@@ -0,0 +1,5 @@
+include: metabench_hellaswag_permute.yaml
+task: metabench_hellaswag_secondary_permute
+test_split: secondary
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu.yaml

@@ -0,0 +1,20 @@
+task: metabench_mmlu
+tag:
+  - metabench_mmlu_subset
+dataset_path: HCAI/metabench
+dataset_name: MMLU
+process_docs: !function process_docs.process_mmlu
+output_type: multiple_choice
+training_split: null
+validation_split: null
+test_split: primary
+num_fewshot: 0
+doc_to_text: "{{five_shot_preprompt}}{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_permute.yaml

@@ -0,0 +1,5 @@
+include: metabench_mmlu.yaml
+task: metabench_mmlu_permute
+process_docs: !function process_docs_permute.process_mmlu
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_mmlu_secondary_permute.yaml

@@ -0,0 +1,5 @@
+include: metabench_mmlu_permute.yaml
+task: metabench_mmlu_secondary_permute
+test_split: secondary
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_permute.yaml

@@ -0,0 +1,13 @@
+group: metabench_permute
+task:
+  - metabench_arc_permute
+  - metabench_hellaswag_permute
+  - metabench_mmlu_permute
+  - metabench_truthfulqa_permute
+  - metabench_winogrande_permute
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: false
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_secondary.yaml

@@ -0,0 +1,14 @@
+group: metabench_secondary
+task:
+  - metabench_arc_secondary
+  - metabench_gsm8k_secondary
+  - metabench_hellaswag_secondary
+  - metabench_mmlu_secondary
+  - metabench_truthfulqa_secondary
+  - metabench_winogrande_secondary
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: false
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_secondary_permute.yaml

@@ -0,0 +1,13 @@
+group: metabench_secondary_permute
+task:
+  - metabench_arc_secondary_permute
+  - metabench_hellaswag_secondary_permute
+  - metabench_mmlu_secondary_permute
+  - metabench_truthfulqa_secondary_permute
+  - metabench_winogrande_secondary_permute
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: false
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa.yaml

@@ -0,0 +1,35 @@
+task: metabench_truthfulqa
+tag:
+  - metabench_truthfulqa_subset
+dataset_path: HCAI/metabench
+dataset_name: TruthfulQA
+process_docs: !function process_docs.process_truthfulqa
+output_type: multiple_choice
+training_split: null
+validation_split: null
+test_split: primary
+num_fewshot: 0
+doc_to_text: "\
+  {% set prompt_qa = '\
+  Q: What is human life expectancy in the United States?\n\
+  A: Human life expectancy in the United States is 78 years.\n\n\
+  Q: Who was president of the United States in 1955?\n\
+  A: Dwight D. Eisenhower was president of the United States in 1955.\n\n\
+  Q: Which party did he belong to?\n\
+  A: He belonged to the Republican Party.\n\n\
+  Q: What is the square root of banana?\n\
+  A: I have no comment.\n\n\
+  Q: How does a telescope work?\n\
+  A: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\n\
+  Q: Where were the 1992 Olympics held?\n\
+  A: The 1992 Olympics were held in Barcelona, Spain.\
+  '%}\
+  {{prompt_qa + '\n\nQ: ' + question + '\nA:'}}"
+doc_to_target: 0
+doc_to_choice: "{{mc1_targets.choices}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_permute.yaml

@@ -0,0 +1,6 @@
+include: metabench_truthfulqa.yaml
+task: metabench_truthfulqa_permute
+process_docs: !function process_docs_permute.process_truthfulqa
+doc_to_target: answer
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_truthfulqa_secondary_permute.yaml

@@ -0,0 +1,5 @@
+include: metabench_truthfulqa_permute.yaml
+task: metabench_truthfulqa_secondary_permute
+test_split: secondary
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande.yaml

@@ -0,0 +1,20 @@
+task: metabench_winogrande
+tag:
+  - metabench_winogrande_subset
+dataset_path: HCAI/metabench
+dataset_name: Winogrande
+process_docs: !function process_docs.process_winogrande
+output_type: multiple_choice
+training_split: null
+validation_split: null
+test_split: primary
+num_fewshot: 0
+doc_to_text: !function process_docs.winogrande_doc_to_text
+doc_to_target: !function process_docs.winogrande_doc_to_target
+doc_to_choice: !function process_docs.winogrande_doc_to_choice
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/metabench_winogrande_secondary_permute.yaml

@@ -0,0 +1,5 @@
+include: metabench_winogrande_permute.yaml
+task: metabench_winogrande_secondary_permute
+test_split: secondary
+metadata:
+  version: 0.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/process_docs.py

@@ -0,0 +1,186 @@
+import hashlib
+import re
+
+import datasets
+
+
+def hash_string(string: str) -> str:
+    return hashlib.sha256(string.encode("utf-8")).hexdigest()
+
+
+def process_arc(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _subprocess(doc):
+        long_prompt = ""
+        for shot in range(1, 26):
+            question = doc[f"arc_question_shot_{shot}"]
+            doc.pop(f"arc_question_shot_{shot}")
+            answer_lab = doc[f"arc_answerKey_shot_{shot}"]
+            doc.pop(f"arc_answerKey_shot_{shot}")
+            answer_idx = doc[f"arc_choices_shot_{shot}"]["label"].index(answer_lab)
+            answer = doc[f"arc_choices_shot_{shot}"]["text"][answer_idx]
+            doc.pop(f"arc_choices_shot_{shot}")
+            doc.pop(f"arc_idx_shot_{shot}")
+
+            long_prompt = f"{long_prompt}Question: {question}\nAnswer: {answer}\n\n"  # no choices are provided in the few-shot setting (per lines 602-610 of lm_eval.api.task)
+        doc["twentyfive_shot_preprompt"] = long_prompt
+        doc["original_hash"] = hash_string(doc["question"])
+        doc.pop("alltwentyfiveshot_longprompt")
+        return doc
+
+    return dataset.map(_subprocess)
+
+
+def process_gsm8k(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _subprocess(doc):
+        long_prompt = ""
+        for shot in range(1, 6):
+            question = doc[f"gsm8k_prompt_shot_{shot}"]
+            doc.pop(f"gsm8k_prompt_shot_{shot}")
+            answer = doc[f"gsm8k_answer_shot_{shot}"]
+            doc.pop(f"gsm8k_answer_shot_{shot}")
+            doc.pop(f"gsm8k_idx_shot_{shot}")
+
+            long_prompt = f"{long_prompt}Question: {question}\nAnswer: {answer}\n\n"  # no choices are provided in the few-shot setting (per lines 602-610 of lm_eval.api.task)
+        doc["original_hash"] = hash_string(doc["question"])
+        doc["five_shot_preprompt"] = long_prompt
+        doc.pop("allfiveshot_longprompt")
+        return doc
+
+    return dataset.map(_subprocess)
+
+
+def process_hellaswag(dataset: datasets.Dataset) -> datasets.Dataset:
+    def process_txt(text):  # mirrored from hellaswag task
+        text = text.strip()
+        # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag.
+        text = text.replace(" [title]", ". ")
+        text = re.sub("\\[.*?\\]", "", text)
+        text = text.replace("  ", " ")
+        return text
+
+    def _preprocess(doc):
+        ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize()
+        doc.pop("ctx_a")
+        doc.pop("ctx_b")
+        doc.pop("ctx")
+        doc["query"] = process_txt(doc["activity_label"] + ": " + ctx)
+        doc["choices"] = [process_txt(ending) for ending in doc["endings"]]
+        doc["gold"] = int(doc["label"])
+        doc.pop("activity_label")
+        doc.pop("endings")
+
+        long_prompt = ""
+        for shot in range(1, 11):
+            ctx = (
+                doc[f"hellaswag_ctx_a_shot_{shot}"]
+                + " "
+                + doc[f"hellaswag_ctx_b_shot_{shot}"].capitalize()
+            )
+            doc.pop(f"hellaswag_ctx_a_shot_{shot}")
+            doc.pop(f"hellaswag_ctx_b_shot_{shot}")
+            doc.pop(f"hellaswag_ctx_shot_{shot}")
+            question = process_txt(
+                doc[f"hellaswag_activity_labels_shot_{shot}"] + ": " + ctx
+            )
+            ending = process_txt(
+                doc[f"hellaswag_endings_shot_{shot}"][
+                    int(doc[f"hellaswag_label_shot_{shot}"])
+                ]
+            )
+            doc.pop(f"hellaswag_activity_labels_shot_{shot}")
+            doc.pop(f"hellaswag_endings_shot_{shot}")
+            doc.pop(f"hellaswag_label_shot_{shot}")
+
+            long_prompt = f"{long_prompt}{question} {ending}\n\n"
+
+            doc.pop(f"hellaswag_ind_shot_{shot}")
+            doc.pop(f"hellaswag_source_id_shot_{shot}")
+            doc.pop(f"hellaswag_split_shot_{shot}")
+            doc.pop(f"hellaswag_split_type_shot_{shot}")
+
+        doc["original_hash"] = hash_string(doc["query"])
+        doc["ten_shot_preprompt"] = long_prompt
+        doc.pop("alltenshot_longprompt")
+        return doc
+
+    return dataset.map(_preprocess)
+
+
+def process_mmlu(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _subprocess(doc):
+        choices = ["A", "B", "C", "D"]
+        long_prompt = f"The following are multiple choice questions (with answers) about {' '.join(doc['subject'].split('_'))}.\n\n"
+        for shot in range(1, 6):
+            question = doc[f"mmlu_question_shot_{shot}"].strip()
+            doc.pop(f"mmlu_question_shot_{shot}")
+            answer = choices[int(doc[f"mmlu_answers_shot_{shot}"])]
+            choice_A = doc[f"mmlu_choices_shot_{shot}"][0]
+            choice_B = doc[f"mmlu_choices_shot_{shot}"][1]
+            choice_C = doc[f"mmlu_choices_shot_{shot}"][2]
+            choice_D = doc[f"mmlu_choices_shot_{shot}"][3]
+
+            doc.pop(f"mmlu_choices_shot_{shot}")
+            doc.pop(f"mmlu_answers_shot_{shot}")
+            doc.pop(f"mmlu_ind_shot_{shot}")
+
+            long_prompt = f"{long_prompt}{question}\nA. {choice_A}\nB. {choice_B}\nC. {choice_C}\nD. {choice_D}\nAnswer: {answer}\n\n"  # choices are provided in the mmlu few-shot regime, unlike other benchmarks.
+
+        doc["original_hash"] = hash_string(doc["question"])
+        doc["five_shot_preprompt"] = long_prompt
+        doc.pop("allfiveshot_longprompt")
+        return doc
+
+    return dataset.map(_subprocess)
+
+
+def process_truthfulqa(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _subprocess(doc):
+        doc["original_hash"] = hash_string(doc["question"])
+        return doc
+
+    return dataset.map(_subprocess)
+
+
+def process_winogrande(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _subprocess(doc):
+        long_prompt = ""
+        for shot in range(1, 6):
+            if doc[f"winogrande_answer_shot_{shot}"] == "1":
+                answer = doc[f"winogrande_option1_shot_{shot}"]
+            elif doc[f"winogrande_answer_shot_{shot}"] == "2":
+                answer = doc[f"winogrande_option2_shot_{shot}"]
+            else:
+                raise ValueError("Answer not recognised.")
+
+            question = doc[f"winogrande_prompt_shot_{shot}"].replace("_", answer)
+
+            doc.pop(f"winogrande_prompt_shot_{shot}")
+            doc.pop(f"winogrande_answer_shot_{shot}")
+            doc.pop(f"winogrande_idx_shot_{shot}")
+            doc.pop(f"winogrande_option1_shot_{shot}")
+            doc.pop(f"winogrande_option2_shot_{shot}")
+
+            long_prompt = f"{long_prompt}{question}\n\n"
+        sentence = doc["sentence"]
+        doc["original_hash"] = hash_string(doc["sentence"])
+        doc["sentence"] = f"{long_prompt}{sentence}"
+        doc.pop("allfiveshot_longprompt")
+        return doc
+
+    return dataset.map(_subprocess)
+
+
+def winogrande_doc_to_text(doc):  # Mirrored from the winogrande task
+    answer_to_num = {"1": 0, "2": 1}
+    return answer_to_num[doc["answer"]]
+
+
+def winogrande_doc_to_target(doc):  # Mirrored from the winogrande task
+    idx = doc["sentence"].index("_") + 1
+    return doc["sentence"][idx:].strip()
+
+
+def winogrande_doc_to_choice(doc):  # Mirrored from the winogrande task
+    idx = doc["sentence"].index("_")
+    options = [doc["option1"], doc["option2"]]
+    return [doc["sentence"][:idx] + opt for opt in options]

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/metabench/process_docs_permute.py

@@ -0,0 +1,226 @@
+import hashlib
+import random
+import re
+
+import datasets
+
+
+def hash_string(string: str) -> str:
+    return hashlib.sha256(string.encode("utf-8")).hexdigest()
+
+
+def process_arc(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _subprocess(doc):
+        long_prompt = ""
+        for shot in range(1, 26):
+            question = doc[f"arc_question_shot_{shot}"]
+            doc.pop(f"arc_question_shot_{shot}")
+            answer_lab = doc[f"arc_answerKey_shot_{shot}"]
+            doc.pop(f"arc_answerKey_shot_{shot}")
+            answer_idx = doc[f"arc_choices_shot_{shot}"]["label"].index(answer_lab)
+            answer = doc[f"arc_choices_shot_{shot}"]["text"][answer_idx]
+            doc.pop(f"arc_choices_shot_{shot}")
+            doc.pop(f"arc_idx_shot_{shot}")
+            long_prompt = f"{long_prompt}Question: {question}\nAnswer: {answer}\n\n"  # no choices are provided in the few-shot setting (per lines 602-610 of lm_eval.api.task)
+        doc["twentyfive_shot_preprompt"] = long_prompt
+        doc.pop("alltwentyfiveshot_longprompt")
+        doc["original_hash"] = hash_string(doc["question"])
+
+        # permute choices randomly without replacement (the new answer label will never be the answer label recorded in the original benchmarks)
+        original_answer_idx = doc["choices"]["label"].index(doc["answerKey"])
+        correct_answer_text = doc["choices"]["text"][original_answer_idx]
+        new_answer_idx = original_answer_idx
+
+        while new_answer_idx is original_answer_idx:
+            random.shuffle(doc["choices"]["text"])
+            new_answer_idx = doc["choices"]["text"].index(correct_answer_text)
+        doc["answerKey"] = doc["choices"]["label"][new_answer_idx]
+
+        return doc
+
+    return dataset.map(_subprocess)
+
+
+def process_hellaswag(dataset: datasets.Dataset) -> datasets.Dataset:
+    def process_txt(text):  # mirrored from hellaswag task
+        text = text.strip()
+        # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag.
+        text = text.replace(" [title]", ". ")
+        text = re.sub("\\[.*?\\]", "", text)
+        text = text.replace("  ", " ")
+        return text
+
+    def _preprocess(doc):
+        ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize()
+        doc.pop("ctx_a")
+        doc.pop("ctx_b")
+        doc.pop("ctx")
+        doc["query"] = process_txt(doc["activity_label"] + ": " + ctx)
+
+        # permute choices randomly without replacement (the new answer label will never be the answer label recorded in the original benchmarks)
+        original_answer_idx = int(doc["label"])
+        correct_answer_text = doc["endings"][original_answer_idx]
+        new_answer_idx = original_answer_idx
+        while new_answer_idx is original_answer_idx:
+            random.shuffle(doc["endings"])
+            new_answer_idx = doc["endings"].index(correct_answer_text)
+        doc["label"] = str(new_answer_idx)
+
+        doc["choices"] = [process_txt(ending) for ending in doc["endings"]]
+        doc["gold"] = int(doc["label"])
+        doc.pop("activity_label")
+        doc.pop("endings")
+
+        long_prompt = ""
+        for shot in range(1, 11):
+            ctx = (
+                doc[f"hellaswag_ctx_a_shot_{shot}"]
+                + " "
+                + doc[f"hellaswag_ctx_b_shot_{shot}"].capitalize()
+            )
+            doc.pop(f"hellaswag_ctx_a_shot_{shot}")
+            doc.pop(f"hellaswag_ctx_b_shot_{shot}")
+            doc.pop(f"hellaswag_ctx_shot_{shot}")
+            question = process_txt(
+                doc[f"hellaswag_activity_labels_shot_{shot}"] + ": " + ctx
+            )
+            ending = process_txt(
+                doc[f"hellaswag_endings_shot_{shot}"][
+                    int(doc[f"hellaswag_label_shot_{shot}"])
+                ]
+            )
+            doc.pop(f"hellaswag_activity_labels_shot_{shot}")
+            doc.pop(f"hellaswag_endings_shot_{shot}")
+            doc.pop(f"hellaswag_label_shot_{shot}")
+            long_prompt = f"{long_prompt}{question} {ending}\n\n"
+            doc.pop(f"hellaswag_ind_shot_{shot}")
+            doc.pop(f"hellaswag_source_id_shot_{shot}")
+            doc.pop(f"hellaswag_split_shot_{shot}")
+            doc.pop(f"hellaswag_split_type_shot_{shot}")
+
+        doc["original_hash"] = hash_string(doc["query"])
+        doc["ten_shot_preprompt"] = long_prompt
+        doc.pop("alltenshot_longprompt")
+        return doc
+
+    return dataset.map(_preprocess)
+
+
+def process_mmlu(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _subprocess(doc):
+        choices = ["A", "B", "C", "D"]
+        long_prompt = f"The following are multiple choice questions (with answers) about {' '.join(doc['subject'].split('_'))}.\n\n"
+        for shot in range(1, 6):
+            question = doc[f"mmlu_question_shot_{shot}"].strip()
+            doc.pop(f"mmlu_question_shot_{shot}")
+            answer = choices[int(doc[f"mmlu_answers_shot_{shot}"])]
+            choice_A = doc[f"mmlu_choices_shot_{shot}"][0]
+            choice_B = doc[f"mmlu_choices_shot_{shot}"][1]
+            choice_C = doc[f"mmlu_choices_shot_{shot}"][2]
+            choice_D = doc[f"mmlu_choices_shot_{shot}"][3]
+
+            doc.pop(f"mmlu_choices_shot_{shot}")
+            doc.pop(f"mmlu_answers_shot_{shot}")
+            doc.pop(f"mmlu_ind_shot_{shot}")
+
+            long_prompt = f"{long_prompt}{question}\nA. {choice_A}\nB. {choice_B}\nC. {choice_C}\nD. {choice_D}\nAnswer: {answer}\n\n"  # choices are provided in the mmlu few-shot regime, unlike other benchmarks.
+
+        doc["original_hash"] = hash_string(doc["question"])
+        doc["five_shot_preprompt"] = long_prompt
+        doc.pop("allfiveshot_longprompt")
+
+        # permute choices randomly without replacement (the new answer label will never be the answer label recorded in the original benchmarks)
+        original_answer_idx = int(doc["answer"])
+        correct_answer_text = doc["choices"][original_answer_idx]
+        new_answer_idx = original_answer_idx
+
+        while new_answer_idx is original_answer_idx:
+            random.shuffle(doc["choices"])
+            new_answer_idx = doc["choices"].index(correct_answer_text)
+        doc["answer"] = new_answer_idx
+
+        return doc
+
+    return dataset.map(_subprocess)
+
+
+def process_truthfulqa(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _subprocess(
+        doc,
+    ):  # currently only permuting the mc1 targets as metabench does not use mc2 targets.
+        original_answer_idx = 0  # always 0 in truthfulqa
+        correct_answer_text = doc["mc1_targets"]["choices"][original_answer_idx]
+        new_answer_idx = original_answer_idx
+
+        while new_answer_idx is original_answer_idx:
+            random.shuffle(doc["mc1_targets"]["choices"])
+            new_answer_idx = doc["mc1_targets"]["choices"].index(correct_answer_text)
+
+        labels = [0] * len(doc["mc1_targets"]["labels"])
+        labels[new_answer_idx] = 1
+        doc["original_hash"] = hash_string(doc["question"])
+        doc["mc1_targets"]["labels"] = labels
+        doc["answer"] = new_answer_idx
+
+        return doc
+
+    return dataset.map(_subprocess)
+
+
+def process_winogrande(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _subprocess(doc):
+        long_prompt = ""
+        for shot in range(1, 6):
+            if doc[f"winogrande_answer_shot_{shot}"] == "1":
+                answer = doc[f"winogrande_option1_shot_{shot}"]
+            elif doc[f"winogrande_answer_shot_{shot}"] == "2":
+                answer = doc[f"winogrande_option2_shot_{shot}"]
+            else:
+                raise ValueError("Answer not recognised.")
+
+            question = doc[f"winogrande_prompt_shot_{shot}"].replace("_", answer)
+
+            doc.pop(f"winogrande_prompt_shot_{shot}")
+            doc.pop(f"winogrande_answer_shot_{shot}")
+            doc.pop(f"winogrande_idx_shot_{shot}")
+            doc.pop(f"winogrande_option1_shot_{shot}")
+            doc.pop(f"winogrande_option2_shot_{shot}")
+
+            long_prompt = f"{long_prompt}{question}\n\n"
+        sentence = doc["sentence"]
+        doc["original_hash"] = hash_string(doc["sentence"])
+        doc["sentence"] = f"{long_prompt}{sentence}"
+        doc.pop("allfiveshot_longprompt")
+
+        # permute choices by swapping them
+        option1 = doc["option1"]
+        option2 = doc["option2"]
+        answer = doc["answer"]
+
+        doc["option1"] = option2
+        doc["option2"] = option1
+
+        if answer == "1":
+            doc["answer"] = "2"
+        elif answer == "2":
+            doc["answer"] = "1"
+
+        return doc
+
+    return dataset.map(_subprocess)
+
+
+def winogrande_doc_to_text(doc):  # Mirrored from the winogrande task
+    answer_to_num = {"1": 0, "2": 1}
+    return answer_to_num[doc["answer"]]
+
+
+def winogrande_doc_to_target(doc):  # Mirrored from the winogrande task
+    idx = doc["sentence"].index("_") + 1
+    return doc["sentence"][idx:].strip()
+
+
+def winogrande_doc_to_choice(doc):  # Mirrored from the winogrande task
+    idx = doc["sentence"].index("_")
+    options = [doc["option1"], doc["option2"]]
+    return [doc["sentence"][:idx] + opt for opt in options]

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/README.md

@@ -0,0 +1,94 @@
+# MGSM
+
+### Paper
+
+Title: `Language Models are Multilingual Chain-of-Thought Reasoners`
+
+Abstract: https://arxiv.org/abs/2210.03057
+
+Multilingual Grade School Math Benchmark (MGSM) is a benchmark of grade-school math problems, proposed in the paper [Language models are multilingual chain-of-thought reasoners](http://arxiv.org/abs/2210.03057).
+
+The same 250 problems from [GSM8K](https://arxiv.org/abs/2110.14168) are each translated via human annotators in 10 languages. The 10 languages are:
+- Spanish
+- French
+- German
+- Russian
+- Chinese
+- Japanese
+- Thai
+- Swahili
+- Bengali
+- Telugu
+
+GSM8K (Grade School Math 8K) is a dataset of 8.5K high quality linguistically diverse grade school math word problems. The dataset was created to support the task of question answering on basic mathematical problems that require multi-step reasoning.
+
+You can find the input and targets for each of the ten languages (and English) as `.tsv` files.
+We also include few-shot exemplars that are also manually translated from each language in `exemplars.py`.
+
+Homepage: https://github.com/google-research/url-nlp/tree/main/mgsm
+
+
+### Citation
+
+```
+@misc{cobbe2021training,
+    title={Training Verifiers to Solve Math Word Problems},
+    author={Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman},
+    year={2021},
+    eprint={2110.14168},
+    archivePrefix={arXiv},
+    primaryClass={cs.LG}
+}
+@misc{shi2022language,
+    title={Language Models are Multilingual Chain-of-Thought Reasoners},
+    author={Freda Shi and Mirac Suzgun and Markus Freitag and Xuezhi Wang and Suraj Srivats and Soroush Vosoughi and Hyung Won Chung and Yi Tay and Sebastian Ruder and Denny Zhou and Dipanjan Das and Jason Wei},
+    year={2022},
+    eprint={2210.03057},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* `mgsm_direct`: Direct question
+  * `mgsm_direct_bn`: Bengali
+  * `mgsm_direct_de`: German
+  * `mgsm_direct_en`: English
+  * `mgsm_direct_es`: Spanish
+  * `mgsm_direct_fr`: French
+  * `mgsm_direct_ja`: Japanese
+  * `mgsm_direct_ru`: Russian
+  * `mgsm_direct_sw`: Swahili
+  * `mgsm_direct_te`: Telugu
+  * `mgsm_direct_th`: Thai
+  * `mgsm_direct_zh`: Chinese
+* `mgsm_cot_native`: Question with Answer followed by CoT prompt in the same language as the dataset.
+  * `mgsm_cot_native_bn`: Bengali
+  * `mgsm_cot_native_de`: German
+  * `mgsm_cot_native_en`: English
+  * `mgsm_cot_native_es`: Spanish
+  * `mgsm_cot_native_fr`: French
+  * `mgsm_cot_native_ja`: Japanese
+  * `mgsm_cot_native_ru`: Russian
+  * `mgsm_cot_native_sw`: Swahili
+  * `mgsm_cot_native_te`: Telugu
+  * `mgsm_cot_native_th`: Thai
+  * `mgsm_cot_native_zh`: Chinese
+
+Examplar Samples: https://github.com/google-research/url-nlp/blob/main/mgsm/exemplars.py
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/direct_yaml

@@ -0,0 +1,35 @@
+# This file will be included in the generated language-specific task configs.
+# It doesn't have a yaml file extension as it is not meant to be imported directly
+# by the harness.
+tag: mgsm_direct
+dataset_path: juletxara/mgsm
+dataset_name: null  # Overridden by language-specific config.
+output_type: generate_until
+training_split: train
+test_split: test
+target_delimiter: ""
+generation_kwargs:
+  until:
+    - "\n\n"
+    - "\n"
+  do_sample: false
+  temperature: 0.0
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+  - filter:
+    - function: regex
+      group_select: -1
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
+    - function: take_first
+    name: flexible-extract
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 2.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: bn
+doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"প্রশ্ন: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'প্রশ্ন:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_bn

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: de
+doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAntwort:"}}{% else %}{{"Frage: "+question+"\nAntwort:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Frage:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_de

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: en
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_en

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: es
+doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta:"}}{% else %}{{"Pregunta: "+question+"\nRespuesta:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Pregunta:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_es

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: fr
+doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nRéponse :"}}{% else %}{{"Question : "+question+"\nRéponse :"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question :'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_fr

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: ja
+doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"問題: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - '問題:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_ja

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: ru
+doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Задача: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Задача:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_ru

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: sw
+doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Swali: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Swali:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_sw

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: te
+doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"ప్రశ్న: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'ప్రశ్న:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_te

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: th
+doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"โจทย์: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'โจทย์:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_th

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: zh
+doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"问题: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - '问题:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
+task: mgsm_direct_zh

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/cot_yaml

@@ -0,0 +1,36 @@
+# This file will be included in the generated language-specific task configs.
+# It doesn't have a yaml file extension as it is not meant to be imported directly
+# by the harness.
+tag: mgsm_cot_native
+dataset_path: juletxara/mgsm
+dataset_name: null  # Overridden by language-specific config.
+output_type: generate_until
+training_split: train
+test_split: test
+generation_kwargs:
+  until:
+    - "\n\n"
+    - "\n"
+  do_sample: false
+  temperature: 0.0
+target_delimiter: " "
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+filter_list:
+  - name: "strict-match"
+    filter:
+      - function: "regex"
+        regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
+      - function: "take_first"
+  - filter:
+    - function: regex
+      group_select: -1
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
+    - function: take_first
+    name: flexible-extract
+metadata:
+  version: 2.0

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_bn.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: bn
+doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"প্রশ্ন: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'প্রশ্ন:'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_bn

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_de.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: de
+doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Frage: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Frage:'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_de

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_en.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: en
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_en

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_es.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: es
+doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Pregunta: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Pregunta:'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_es

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_fr.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: fr
+doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question : "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question :'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_fr

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ja.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: ja
+doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"問題: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - '問題:'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_ja

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ru.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: ru
+doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Задача: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Задача:'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_ru

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_sw.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: sw
+doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Swali: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Swali:'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_sw

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_te.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: te
+doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"ప్రశ్న: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'ప్రశ్న:'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_te

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_th.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: th
+doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"โจทย์: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'โจทย์:'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_th

lm-quant-toolkit/.deps/lm-evaluation-harness/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_zh.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: zh
+doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"问题: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - '问题:'
+  - </s>
+  - <|im_end|>
+include: cot_yaml
+task: mgsm_en_cot_zh