diff --git a/llm-awq/tinychat/serve/README.md b/llm-awq/tinychat/serve/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d5c92eb7f56b61b6ad3faef6bb43a3bdbff8bb75 --- /dev/null +++ b/llm-awq/tinychat/serve/README.md @@ -0,0 +1,27 @@ +## Gradio demo: VILA with TinyChat + +We provide scripts for building your own gradio server to run VILA models with TinyChat. Please run the following commands to launch the server. + +#### Launch a controller +```bash +python -m tinychat.serve.controller --host 0.0.0.0 --port 10000 +``` + +#### Launch gradio web server. +```bash +python -m tinychat.serve.gradio_web_server --controller http://localhost:10000 --model-list-mode reload --share --auto-pad-image-token +``` +After launching this script, the web interface will be served on your machine and you can access it with a public URL (or localhost URL). + +#### Launch a model worker + +```bash +python -m tinychat.serve.model_worker_new --host 0.0.0.0 --controller http://localhost:10000 --port 40000 --worker http://localhost:40000 --model-path --quant-path +# Please change tinychat.serve.model_worker_new to tinychat.serve.model_worker if you want to serve VILA rather than VILA-1.5 +``` + +Note: You can launch multiple model workers onto the same web server. And please remember to specify different ports for each model worker. + +### Acknowlegement + +This demo is inspired by [LLaVA](https://github.com/haotian-liu/LLaVA). We thank LLaVA for providing an elegant way to build the Gradio Web UI. diff --git a/llm-awq/tinychat/serve/controller.py b/llm-awq/tinychat/serve/controller.py new file mode 100644 index 0000000000000000000000000000000000000000..596f69e8b9b1f65200e75ac086cedbec8615e5c0 --- /dev/null +++ b/llm-awq/tinychat/serve/controller.py @@ -0,0 +1,325 @@ +# Modified from https://github.com/haotian-liu/LLaVA +# Copyright 2023 Haotian Liu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A controller manages distributed workers. +It sends worker addresses to clients. +""" +import argparse +import asyncio +import dataclasses +from enum import Enum, auto +import json +import logging +import time +from typing import List, Union +import threading + +from fastapi import FastAPI, Request +from fastapi.responses import StreamingResponse +import numpy as np +import requests +import uvicorn + +from tinychat.utils.constants import CONTROLLER_HEART_BEAT_EXPIRATION +from tinychat.utils.log_utils import build_logger, server_error_msg + + +logger = build_logger("controller", "controller.log") + + +class DispatchMethod(Enum): + LOTTERY = auto() + SHORTEST_QUEUE = auto() + + @classmethod + def from_str(cls, name): + if name == "lottery": + return cls.LOTTERY + elif name == "shortest_queue": + return cls.SHORTEST_QUEUE + else: + raise ValueError(f"Invalid dispatch method") + + +@dataclasses.dataclass +class WorkerInfo: + model_names: List[str] + speed: int + queue_length: int + check_heart_beat: bool + last_heart_beat: str + + +def heart_beat_controller(controller): + while True: + time.sleep(CONTROLLER_HEART_BEAT_EXPIRATION) + controller.remove_stable_workers_by_expiration() + + +class Controller: + def __init__(self, dispatch_method: str): + # Dict[str -> WorkerInfo] + self.worker_info = {} + self.dispatch_method = DispatchMethod.from_str(dispatch_method) + + self.heart_beat_thread = threading.Thread( + target=heart_beat_controller, args=(self,) + ) + self.heart_beat_thread.start() + + logger.info("Init controller") + + def register_worker( + self, worker_name: str, check_heart_beat: bool, worker_status: dict + ): + if worker_name not in self.worker_info: + logger.info(f"Register a new worker: {worker_name}") + else: + logger.info(f"Register an existing worker: {worker_name}") + + if not worker_status: + worker_status = self.get_worker_status(worker_name) + if not worker_status: + return False + + self.worker_info[worker_name] = WorkerInfo( + worker_status["model_names"], + worker_status["speed"], + worker_status["queue_length"], + check_heart_beat, + time.time(), + ) + + logger.info(f"Register done: {worker_name}, {worker_status}") + return True + + def get_worker_status(self, worker_name: str): + try: + r = requests.post(worker_name + "/worker_get_status", timeout=5) + except requests.exceptions.RequestException as e: + logger.error(f"Get status fails: {worker_name}, {e}") + return None + + if r.status_code != 200: + logger.error(f"Get status fails: {worker_name}, {r}") + return None + + return r.json() + + def remove_worker(self, worker_name: str): + del self.worker_info[worker_name] + + def refresh_all_workers(self): + old_info = dict(self.worker_info) + self.worker_info = {} + + for w_name, w_info in old_info.items(): + if not self.register_worker(w_name, w_info.check_heart_beat, None): + logger.info(f"Remove stale worker: {w_name}") + + def list_models(self): + model_names = set() + + for w_name, w_info in self.worker_info.items(): + model_names.update(w_info.model_names) + + return list(model_names) + + def get_worker_address(self, model_name: str): + if self.dispatch_method == DispatchMethod.LOTTERY: + worker_names = [] + worker_speeds = [] + for w_name, w_info in self.worker_info.items(): + if model_name in w_info.model_names: + worker_names.append(w_name) + worker_speeds.append(w_info.speed) + worker_speeds = np.array(worker_speeds, dtype=np.float32) + norm = np.sum(worker_speeds) + if norm < 1e-4: + return "" + worker_speeds = worker_speeds / norm + if True: # Directly return address + pt = np.random.choice(np.arange(len(worker_names)), p=worker_speeds) + worker_name = worker_names[pt] + return worker_name + + # Check status before returning + while True: + pt = np.random.choice(np.arange(len(worker_names)), p=worker_speeds) + worker_name = worker_names[pt] + + if self.get_worker_status(worker_name): + break + else: + self.remove_worker(worker_name) + worker_speeds[pt] = 0 + norm = np.sum(worker_speeds) + if norm < 1e-4: + return "" + worker_speeds = worker_speeds / norm + continue + return worker_name + elif self.dispatch_method == DispatchMethod.SHORTEST_QUEUE: + worker_names = [] + worker_qlen = [] + for w_name, w_info in self.worker_info.items(): + if model_name in w_info.model_names: + worker_names.append(w_name) + worker_qlen.append(w_info.queue_length / w_info.speed) + if len(worker_names) == 0: + return "" + min_index = np.argmin(worker_qlen) + w_name = worker_names[min_index] + self.worker_info[w_name].queue_length += 1 + logger.info( + f"names: {worker_names}, queue_lens: {worker_qlen}, ret: {w_name}" + ) + return w_name + else: + raise ValueError(f"Invalid dispatch method: {self.dispatch_method}") + + def receive_heart_beat(self, worker_name: str, queue_length: int): + if worker_name not in self.worker_info: + logger.info(f"Receive unknown heart beat. {worker_name}") + return False + + self.worker_info[worker_name].queue_length = queue_length + self.worker_info[worker_name].last_heart_beat = time.time() + logger.info(f"Receive heart beat. {worker_name}") + return True + + def remove_stable_workers_by_expiration(self): + expire = time.time() - CONTROLLER_HEART_BEAT_EXPIRATION + to_delete = [] + for worker_name, w_info in self.worker_info.items(): + if w_info.check_heart_beat and w_info.last_heart_beat < expire: + to_delete.append(worker_name) + + for worker_name in to_delete: + self.remove_worker(worker_name) + + def worker_api_generate_stream(self, params): + worker_addr = self.get_worker_address(params["model"]) + if not worker_addr: + logger.info(f"no worker: {params['model']}") + ret = { + "text": server_error_msg, + "error_code": 2, + } + yield json.dumps(ret).encode() + b"\0" + + try: + response = requests.post( + worker_addr + "/worker_generate_stream", + json=params, + stream=True, + timeout=5, + ) + for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"): + if chunk: + yield chunk + b"\0" + except requests.exceptions.RequestException as e: + logger.info(f"worker timeout: {worker_addr}") + ret = { + "text": server_error_msg, + "error_code": 3, + } + yield json.dumps(ret).encode() + b"\0" + + # Let the controller act as a worker to achieve hierarchical + # management. This can be used to connect isolated sub networks. + def worker_api_get_status(self): + model_names = set() + speed = 0 + queue_length = 0 + + for w_name in self.worker_info: + worker_status = self.get_worker_status(w_name) + if worker_status is not None: + model_names.update(worker_status["model_names"]) + speed += worker_status["speed"] + queue_length += worker_status["queue_length"] + + return { + "model_names": list(model_names), + "speed": speed, + "queue_length": queue_length, + } + + +app = FastAPI() + + +@app.post("/register_worker") +async def register_worker(request: Request): + data = await request.json() + controller.register_worker( + data["worker_name"], data["check_heart_beat"], data.get("worker_status", None) + ) + + +@app.post("/refresh_all_workers") +async def refresh_all_workers(): + models = controller.refresh_all_workers() + + +@app.post("/list_models") +async def list_models(): + models = controller.list_models() + return {"models": models} + + +@app.post("/get_worker_address") +async def get_worker_address(request: Request): + data = await request.json() + addr = controller.get_worker_address(data["model"]) + return {"address": addr} + + +@app.post("/receive_heart_beat") +async def receive_heart_beat(request: Request): + data = await request.json() + exist = controller.receive_heart_beat(data["worker_name"], data["queue_length"]) + return {"exist": exist} + + +@app.post("/worker_generate_stream") +async def worker_api_generate_stream(request: Request): + params = await request.json() + generator = controller.worker_api_generate_stream(params) + return StreamingResponse(generator) + + +@app.post("/worker_get_status") +async def worker_api_get_status(request: Request): + return controller.worker_api_get_status() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="localhost") + parser.add_argument("--port", type=int, default=21001) + parser.add_argument( + "--dispatch-method", + type=str, + choices=["lottery", "shortest_queue"], + default="shortest_queue", + ) + args = parser.parse_args() + logger.info(f"args: {args}") + + controller = Controller(args.dispatch_method) + uvicorn.run(app, host=args.host, port=args.port, log_level="info") diff --git a/llm-awq/tinychat/stream_generators/NVILA_stream_gen.py b/llm-awq/tinychat/stream_generators/NVILA_stream_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..d747260f413badefb32d6111f102ecab553c8ba8 --- /dev/null +++ b/llm-awq/tinychat/stream_generators/NVILA_stream_gen.py @@ -0,0 +1,176 @@ +import torch +import gc +import time +from typing import Optional + +from .llava_stream_gen import prepare_logits_processor + +context_tokens = 0 +context_time = 0.0 +total_tokens = 0 +generation_time_list = [] + + +@torch.inference_mode() +def NVILAStreamGenerator( + model, + gen_params, + input: str, + media=None, + media_cfg=None, + start_pos: int = 0, + device: str = "cuda:0", + stream_interval: int = 2, + echo: bool = False, + stop_token_ids=[], + image_tensor: Optional[torch.FloatTensor] = None, + chunk_prefilling: bool = False, + quant_llm: bool = False, +): + if chunk_prefilling and start_pos != 0: + input = "<|im_start|>" + input + input_ids = model.tokenizer(input)["input_ids"] + output_ids = list(input_ids) + input_echo_len = len(output_ids) + len_input = len(input) + if gen_params.top_k <= 0: + top_k = gen_params.n_vocab + else: + top_k = gen_params.top_k + logits_processor = prepare_logits_processor( + gen_params.temp, gen_params.repeat_penalty, gen_params.top_p, top_k + ) + past_key_values = out = None + stop_token_ids.append(model.tokenizer.eos_token_id) + max_new_tokens = gen_params.n_predict + + for i in range(max_new_tokens): + torch.cuda.synchronize() + t_st = time.time() + + if i == 0: + inputs = torch.as_tensor([input_ids], device=device) + else: + inputs = torch.as_tensor([[token]], device=device) + out, length = model.stream_gen( + input_ids=inputs, + media=media, + media_cfg=media_cfg, + start_pos=start_pos, + chunk_prefilling=chunk_prefilling, + quant_llm=quant_llm, + ) + start_pos += length + logits = out + torch.cuda.synchronize() + t_ed = time.time() + media = None + media_cfg = None + if torch.sum(torch.isinf(logits)): + print( + "{a} of {b}".format( + a=torch.sum(torch.isinf(logits)).item(), b=logits.numel() + ) + ) + print("{},{}".format(torch.max(logits), torch.min(logits))) + # Processing the logits + if logits_processor: + if gen_params.repeat_penalty > 1.0: + tmp_output_ids = torch.as_tensor([output_ids], device=logits.device) + # tmp_output_ids = output_ids[0].unsqueeze(0) + else: + tmp_output_ids = None + last_token_logits = logits_processor(tmp_output_ids, logits[:, -1, :])[0] + else: + last_token_logits = logits[:, -1, :] + if gen_params.temp < 1e-5 or gen_params.top_p < 1e-8: # greedy + token = int(torch.argmax(last_token_logits)) + else: + probs = torch.softmax(last_token_logits.float(), dim=-1) + if torch.any(torch.isinf(probs)) or torch.any(torch.isnan(probs)): + print( + "[Error] Invalid probabilities detected (Inf/Nan exists). Saving the tensor and exiting..." + ) + torch.save(last_token_logits, "last_token_logits.pt") + exit() + token = int(torch.multinomial(probs, num_samples=1)) + output_ids.append(token) + + global context_time + global context_tokens + global total_tokens + global generation_time_list + if i == 0: + context_time = t_ed - t_st + context_tokens = length + generation_time_list = [] + else: + generation_time_list.append(t_ed - t_st) + + if token in stop_token_ids: + stopped = True + else: + stopped = False + + if i % stream_interval == 0 or i == max_new_tokens - 1 or stopped: + if echo: + tmp_output_ids = output_ids + rfind_start = len_input + else: + tmp_output_ids = output_ids[input_echo_len:] + rfind_start = 0 + + output = model.tokenizer.decode( + tmp_output_ids, + skip_special_tokens=True, + spaces_between_special_tokens=False, + ) + + partially_stopped = False + + # prevent yielding partial stop sequence + if not partially_stopped: + yield { + "text": output, + "usage": { + "prompt_tokens": input_echo_len, + "completion_tokens": i, + "total_tokens": input_echo_len + i, + }, + "finish_reason": None, + "timing": None, + } + + if stopped: + break + + # finish stream event, which contains finish reason + if i == max_new_tokens - 1: + finish_reason = "length" + elif stopped: + finish_reason = "stop" + else: + finish_reason = None + + total_tokens = context_tokens + len(generation_time_list) + yield { + "text": output, + "usage": { + "prompt_tokens": input_echo_len, + "completion_tokens": i, + "total_tokens": input_echo_len + i, + }, + "finish_reason": finish_reason, + "timing": { + "context_tokens": context_tokens, + "context_time": context_time, + "total_tokens": total_tokens, + "generation_time_list": generation_time_list, + }, + } + + del past_key_values, out + gc.collect() + torch.cuda.empty_cache() + + # return context_tokens, context_time, total_tokens, generation_time_list diff --git a/llm-awq/tinychat/stream_generators/__init__.py b/llm-awq/tinychat/stream_generators/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bd581695cb7fd2f838c4b55645bc10362fb602f6 --- /dev/null +++ b/llm-awq/tinychat/stream_generators/__init__.py @@ -0,0 +1 @@ +from .stream_gen import * diff --git a/llm-awq/tinychat/stream_generators/llava_stream_gen.py b/llm-awq/tinychat/stream_generators/llava_stream_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..208df0707b907e082baa650a1279d82452bab354 --- /dev/null +++ b/llm-awq/tinychat/stream_generators/llava_stream_gen.py @@ -0,0 +1,301 @@ +import torch +import gc +import time +from typing import Optional + +import tinychat.utils.constants + +from transformers.generation.logits_process import ( + LogitsProcessorList, + RepetitionPenaltyLogitsProcessor, + TemperatureLogitsWarper, + TopKLogitsWarper, + TopPLogitsWarper, +) + +# from llava.constants import ( +# IMAGE_TOKEN_INDEX, +# ) + +context_tokens = 0 +context_time = 0.0 +total_tokens = 0 +generation_time_list = [] + + +def prepare_logits_processor( + temperature: float, + repetition_penalty: float, + top_p: float, + top_k: int, + min_tokens_to_keep: int = 1, +) -> LogitsProcessorList: + processor_list = LogitsProcessorList() + # TemperatureLogitsWarper doesn't accept 0.0, 1.0 makes it a no-op so we skip two cases. + if temperature >= 1e-5 and temperature != 1.0: + processor_list.append(TemperatureLogitsWarper(temperature)) + # Removed for the newest version of VILA + # if repetition_penalty > 1.0: + # processor_list.append(RepetitionPenaltyLogitsProcessor(repetition_penalty)) + if 1e-8 <= top_p < 1.0: + processor_list.append(TopPLogitsWarper(top_p)) + if top_k > 0: + processor_list.append( + TopKLogitsWarper(top_k=top_k, min_tokens_to_keep=min_tokens_to_keep) + ) + return processor_list + + +# This function is inspired by https://github.com/haotian-liu/LLaVA/blob/main/llava/mm_utils.py#L185 +def tokenizer_image_token( + prompt, + tokenizer, + image_token_index=tinychat.utils.constants.LLAVA_DEFAULT_IMAGE_TOKEN_IDX, + return_tensors=None, +): + prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split("")] + + def insert_separator(X, sep): + return [ele for sublist in zip(X, [sep] * len(X)) for ele in sublist][:-1] + + input_ids = [] + offset = 0 + if ( + len(prompt_chunks) > 0 + and len(prompt_chunks[0]) > 0 + and prompt_chunks[0][0] == tokenizer.bos_token_id + ): + offset = 1 + input_ids.append(prompt_chunks[0][0]) + + for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)): + input_ids.extend(x[offset:]) + + if return_tensors is not None: + if return_tensors == "pt": + return torch.tensor(input_ids, dtype=torch.long) + raise ValueError(f"Unsupported tensor type: {return_tensors}") + return input_ids + + +@torch.inference_mode() +def LlavaStreamGenerator( + model, + tokenizer, + input: str, + start_pos: int, + gen_params: dict, + device: str = "cuda:0", + stream_interval: int = 1, + echo: bool = False, + stop_token_ids=[], + image_tensor: Optional[torch.FloatTensor] = None, + chunk_prefilling: bool = False, +): + if chunk_prefilling and start_pos != 0: # USER:2,11889 while USER:3148,1001 + input = "" + input + input_ids = ( + tokenizer_image_token( + input, + tokenizer, + tinychat.utils.constants.LLAVA_DEFAULT_IMAGE_TOKEN_IDX, + return_tensors="pt", + ) + .unsqueeze(0) + .to(device) + ) + if chunk_prefilling and start_pos != 0: + input_ids = input_ids[ + :, 2: + ] # tokenizer will add a at the beginning, so to delete it + special_token = "" in input + input_echo_len = len(input_ids) + output_ids = list(input_ids) + len_input = len(input) + if gen_params.top_k <= 0: + top_k = gen_params.n_vocab + else: + top_k = gen_params.top_k + logits_processor = prepare_logits_processor( + gen_params.temp, gen_params.repeat_penalty, gen_params.top_p, top_k + ) + + past_key_values = out = None + stop_token_ids.append(tokenizer.eos_token_id) + max_new_tokens = gen_params.n_predict + + batch_size = 1 # TODO: support multi-batch + position_ids = [ + torch.arange( + start_pos, start_pos + input_ids.numel(), dtype=torch.long, device=device + ) + for i in range(batch_size) + ] + position_ids = torch.stack(position_ids) + + for i in range(max_new_tokens): + torch.cuda.synchronize() + t_st = time.time() + + if i == 0: + # inputs = torch.as_tensor([input_ids], device=device) + inputs = input_ids + else: + position_ids = (position_ids[:, -1] + 1).reshape( + 1, 1 + ) # [Important] fixed the bug of positions + inputs = torch.as_tensor([[token]], device=device) + + attention_mask = torch.ones(size=inputs.shape, dtype=torch.int, device=device) + + if ( + "llama" not in model.__class__.__name__.lower() + and "mpt" not in model.__class__.__name__.lower() + and "falcon" not in model.__class__.__name__.lower() + and "llava" not in model.__class__.__name__.lower() + # and "vila" not in model.__class__.__name__.lower() # VILA model reuses the model class of LLaVA + ): + if i == 0: # Context Stage + out = model( + input_ids=inputs, + attention_mask=attention_mask, + position_ids=position_ids, + use_cache=True, + output_attentions=False, + output_hidden_states=False, + images=image_tensor, + return_dict=True, + # special_token=special_token, + ) + logits = out.logits + past_key_values = out.past_key_values + else: + out = model( + input_ids=inputs, + attention_mask=attention_mask, + position_ids=position_ids, + use_cache=True, + past_key_values=past_key_values, + output_attentions=False, + output_hidden_states=False, + images=image_tensor, + return_dict=True, + # special_token=special_token, + ) + logits = out.logits + past_key_values = out.past_key_values + else: + out = model( + input_ids=inputs, + start_pos=start_pos, + images=image_tensor, + position_ids=position_ids, + attention_mask=attention_mask, + special_token=special_token, + chunk_prefilling=chunk_prefilling, + ) + start_pos += ( + inputs.shape[1] + 195 * torch.sum(inputs[0] == IMAGE_TOKEN_INDEX).item() + ) + logits = out + torch.cuda.synchronize() + t_ed = time.time() + + # Processing the logits + if logits_processor: + if gen_params.repeat_penalty > 1.0: + # tmp_output_ids = torch.as_tensor([output_ids], device=logits.device) + tmp_output_ids = output_ids[0].unsqueeze(0) + else: + tmp_output_ids = None + last_token_logits = logits_processor(tmp_output_ids, logits[:, -1, :])[0] + else: + last_token_logits = logits[:, -1, :] + if gen_params.temp < 1e-5 or gen_params.top_p < 1e-8: # greedy + token = int(torch.argmax(last_token_logits)) + print(token) + else: + probs = torch.softmax(last_token_logits, dim=-1) + token = int(torch.multinomial(probs, num_samples=1)) + output_ids.append(token) + + global context_time + global context_tokens + global total_tokens + global generation_time_list + if i == 0: + context_time = t_ed - t_st + context_tokens = ( + inputs.shape[1] + 195 * torch.sum(inputs[0] == IMAGE_TOKEN_INDEX).item() + ) + generation_time_list = [] + else: + generation_time_list.append(t_ed - t_st) + + if token in stop_token_ids: + stopped = True + else: + stopped = False + + if i % stream_interval == 0 or i == max_new_tokens - 1 or stopped: + if echo: + tmp_output_ids = output_ids + rfind_start = len_input + else: + tmp_output_ids = output_ids[input_echo_len:] + rfind_start = 0 + + output = tokenizer.decode( + tmp_output_ids, + skip_special_tokens=True, + spaces_between_special_tokens=False, + ) + + partially_stopped = False + + # prevent yielding partial stop sequence + if not partially_stopped: + yield { + "text": output, + "usage": { + "prompt_tokens": input_echo_len, + "completion_tokens": i, + "total_tokens": input_echo_len + i, + }, + "finish_reason": None, + "timing": None, + } + + if stopped: + break + + # finish stream event, which contains finish reason + if i == max_new_tokens - 1: + finish_reason = "length" + elif stopped: + finish_reason = "stop" + else: + finish_reason = None + + total_tokens = context_tokens + len(generation_time_list) + yield { + "text": output, + "usage": { + "prompt_tokens": input_echo_len, + "completion_tokens": i, + "total_tokens": input_echo_len + i, + }, + "finish_reason": finish_reason, + "timing": { + "context_tokens": context_tokens, + "context_time": context_time, + "total_tokens": total_tokens, + "generation_time_list": generation_time_list, + }, + } + + del past_key_values, out + gc.collect() + torch.cuda.empty_cache() + + # return context_tokens, context_time, total_tokens, generation_time_list diff --git a/llm-awq/tinychat/utils/constants.py b/llm-awq/tinychat/utils/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..8ada509238306072dd9dcd8f16c21737c6407704 --- /dev/null +++ b/llm-awq/tinychat/utils/constants.py @@ -0,0 +1,26 @@ +import torch + + +def init(): + global max_seq_len, max_batch_size, llama_multiple_of, mem_efficient_load + max_seq_len = 8192 + max_batch_size = 1 + llama_multiple_of = 256 + mem_efficient_load = False # Whether to load the checkpoint in a layer-wise manner. Activate this if you are facing OOM issues on edge devices (e.g., Jetson Orin). + + # LLaVA Constants + global LLAVA_IGNORE_INDEX, LLAVA_DEFAULT_IMAGE_TOKEN, LLAVA_DEFAULT_IMAGE_TOKEN_IDX, LLAVA_DEFAULT_IMAGE_PATCH_TOKEN, LLAVA_DEFAULT_IMAGE_PATCH_TOKEN_IDX, LLAVA_DEFAULT_IM_START_TOKEN, LLAVA_DEFAULT_IM_END_TOKEN, LLAVA_DEFAULT_IM_TOKEN_PLACE_HOLDER, AUTO_FILL_IM_TOKEN_HOLDER + LLAVA_IGNORE_INDEX = -100 + LLAVA_DEFAULT_IMAGE_TOKEN = "" + LLAVA_DEFAULT_IMAGE_TOKEN_IDX = -200 + LLAVA_DEFAULT_IMAGE_PATCH_TOKEN = "" + LLAVA_DEFAULT_IMAGE_PATCH_TOKEN_IDX = 32000 + LLAVA_DEFAULT_IM_START_TOKEN = "" + LLAVA_DEFAULT_IM_END_TOKEN = "" + LLAVA_DEFAULT_IM_TOKEN_PLACE_HOLDER = "" + AUTO_FILL_IM_TOKEN_HOLDER = "" + + # gradio UI + global CONTROLLER_HEART_BEAT_EXPIRATION, WORKER_HEART_BEAT_INTERVAL + CONTROLLER_HEART_BEAT_EXPIRATION = 30 + WORKER_HEART_BEAT_INTERVAL = 15 diff --git a/llm-awq/tinychat/utils/input_metadata.py b/llm-awq/tinychat/utils/input_metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..90c90ec40afe35cc346d74789aae1aada2d83366 --- /dev/null +++ b/llm-awq/tinychat/utils/input_metadata.py @@ -0,0 +1,102 @@ +# File authors: Haotian Tang, Shang Yang, Yujun Lin, Song Han +# @article{lin2024awq, +# title={AWQ: Activation-aware Weight Quantization for On-Device LLM Compression and Acceleration}, +# author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang, Shang and Chen, Wei-Ming and Wang, Wei-Chen and Xiao, Guangxuan and Dang, Xingyu and Gan, Chuang and Han, Song}, +# journal={Proceedings of Machine Learning and Systems}, +# volume={6}, +# pages={87--100}, +# year={2024} +# } + +import torch + + +class ActivationBuffer: + """ + Pre-allocated Buffer for activation in the siglip model. + + Args: + model: The input model + batched_seq_len: The batched sequence length. Sum of all the sequence lengths in the batch. + """ + + def __init__(self, model): + self.model_class = model.__class__.__name__ + + if self.model_class == "SiglipEncoder": + self.model_dtype = model.layers[0].self_attn.k_proj.weight.dtype + + self.device = "cuda" + assert self.model_class in [ + "SiglipEncoder", + ], f"model_class: {self.model_class} is currently not supported." + assert ( + self.model_dtype == torch.float16 + ), f"model_dtype is expected to be fp16. Current: {self.model_dtype}." + + self.intermediate_size = model.config.intermediate_size + self.hidden_size = model.config.hidden_size + + def allocate_activation_buffer(self, batched_seq_len): + if self.model_class == "SiglipEncoder": + self.__allocate_activation_buffer_siglip(batched_seq_len) + else: + raise NotImplementedError( + f"model_class: {self.model_class} is currently not supported." + ) + + def __allocate_activation_buffer_siglip(self, batched_seq_len): + # Allocate fp16 activation buffer. + self.act_buffer = torch.empty( + (batched_seq_len * max(self.hidden_size * 3, 2 * self.intermediate_size)), + device=self.device, + dtype=torch.float16, + ) + self.qkv_proj_act_buffer = self.act_buffer[ + : batched_seq_len * self.hidden_size * 3 + ].view( + batched_seq_len, self.hidden_size * 3 + ) # qkv + + self.in_out_fc2_act_buffer = self.act_buffer[ + : batched_seq_len * self.hidden_size + ].view( + batched_seq_len, self.hidden_size + ) # LN1, Wo_out, LN2, all_out + + self.fc1_buffer = self.act_buffer[ + : batched_seq_len * self.intermediate_size + ].view(batched_seq_len, self.intermediate_size) + self.actfn_buffer = self.act_buffer[ + batched_seq_len + * self.intermediate_size : 2 + * batched_seq_len + * self.intermediate_size + ].view(batched_seq_len, self.intermediate_size) + + # Allocate quantized activation buffer. + self.quantized_act_buffer = torch.empty( + (batched_seq_len * max(self.hidden_size, self.intermediate_size)), + device=self.device, + dtype=torch.int8, + ) + self.quantized_hidden_states_buffer = self.quantized_act_buffer[ + : batched_seq_len * self.hidden_size + ].view( + batched_seq_len, self.hidden_size + ) # Wo_in, + self.quantized_mlp_act_buffer = self.quantized_act_buffer[ + : batched_seq_len * self.intermediate_size + ].view(batched_seq_len, self.intermediate_size) + + # per token + self.quantized_scale_buffer = torch.empty( + (batched_seq_len), device=self.device, dtype=torch.float16 + ) + + # For faster act-quant implementation + self.tmp = torch.empty( + (batched_seq_len * self.intermediate_size), + device=self.device, + dtype=torch.float16, + ) diff --git a/llm-awq/tinychat/utils/load_quant.py b/llm-awq/tinychat/utils/load_quant.py new file mode 100644 index 0000000000000000000000000000000000000000..f6bc3c660751d58aedc832cc9905028a5d596d4d --- /dev/null +++ b/llm-awq/tinychat/utils/load_quant.py @@ -0,0 +1,171 @@ +import gc +import os +import re +from typing import Union, List + +import torch +import torch.nn as nn +from transformers import AutoModelForCausalLM +from accelerate import init_empty_weights, load_checkpoint_and_dispatch +from awq.quantize.quantizer import real_quantize_model_weight +from awq.quantize.qmodule import WQLinear +from tqdm import tqdm + +import tinychat.utils.constants + +version_message = """ +[Warning] The awq quantized checkpoint seems to be in v1 format. +If the model cannot be loaded successfully, please use the latest awq library to re-quantized the model, or repack the current checkpoint with tinychat/offline-weight-repacker.py +""" + + +def ckpt_version_check(quant_path): + if not quant_path.endswith("v2.pt"): + print(version_message) + + +def mem_efficient_load_checkpoint( + model: nn.Module, + ckpts_folder: Union[str, os.PathLike], +): + checkpoint_files = [ + ckpts_folder + "/" + f for f in os.listdir(ckpts_folder) if f.endswith(".pt") + ] + + # Check if the ckpts match the model + model_keys = sorted((list(model.state_dict().keys()))) + suffix = r"\.pt$" + ckpt_keys = sorted( + [re.sub(suffix, "", f) for f in os.listdir(ckpts_folder) if f.endswith(".pt")] + ) + assert len(model_keys) == len( + ckpt_keys + ), f"The number of checkpoint files do not match the model. \n Model has {len(model_keys)} keys, while finding {len(ckpt_keys)} checkpoint files in the folder." + for key1, key2 in zip(model_keys, ckpt_keys): + assert ( + key1 == key2 + ), f"The checkpoint files do not match the model. \nmodel key {key1} != checkpoint key {key2}" + + with tqdm(total=len(checkpoint_files)) as pbar: + pbar.set_description("Loading checkpoint shards") + for checkpoint_file in checkpoint_files: + checkpoint = torch.load(checkpoint_file, map_location=torch.device("cpu")) + model.load_state_dict(checkpoint, strict=False) + # Force Python to clean up. + del checkpoint + gc.collect() + pbar.update(1) + return model + + +def load_awq_model(model, checkpoint, w_bit, group_size, device): + q_config = {"zero_point": True, "q_group_size": group_size} + real_quantize_model_weight(model, w_bit, q_config, init_only=True) + + if hasattr(model.config, "tie_encoder_decoder"): + model.config.tie_encoder_decoder = False + if hasattr(model.config, "tie_word_embeddings"): + model.config.tie_word_embeddings = False + if tinychat.utils.constants.mem_efficient_load: + assert os.path.isdir( + checkpoint + ), "You are in mem_efficient_load mode. \n Please set --load_quant the path to the folder containing all checkpoint files." + model = mem_efficient_load_checkpoint( + model, + checkpoint, + ).to(device) + else: + ckpt_version_check(checkpoint) + pbar = tqdm(range(1)) + pbar.set_description("Loading checkpoint") + for i in pbar: + model = load_checkpoint_and_dispatch( + model, + checkpoint, + no_split_module_classes=[ + "OPTDecoderLayer", + "LlamaDecoderLayer", + "BloomBlock", + "MPTBlock", + "DecoderLayer", + "CLIPEncoderLayer", + ], + ).to(device) + return model + + +def make_quant_linear(module, names, w_bit, groupsize, device, name=""): + if isinstance(module, WQLinear): + return + for attr in dir(module): + tmp = getattr(module, attr) + name1 = name + "." + attr if name != "" else attr + if name1 in names: + delattr(module, attr) + setattr( + module, + attr, + WQLinear( + w_bit, + groupsize, + tmp.in_features, + tmp.out_features, + tmp.bias is not None, + device, + dtype=tmp.weight.dtype, + ), + ) + for name1, child in module.named_children(): + make_quant_linear( + child, + names, + w_bit, + groupsize, + device, + name + "." + name1 if name != "" else name1, + ) + + +def find_layers(module, layers=[nn.Linear], name=""): + if type(module) in layers: + return {name: module} + res = {} + for name1, child in module.named_children(): + res.update( + find_layers( + child, layers=layers, name=name + "." + name1 if name != "" else name1 + ) + ) + return res + + +def load_awq_llama_fast(model, checkpoint, w_bit, group_size, device): + layers = find_layers(model) + for name in ["lm_head"]: + if name in layers: + del layers[name] + make_quant_linear(model, layers, w_bit, group_size, device) + del layers + + if tinychat.utils.constants.mem_efficient_load: + # TODO: mem-efficient load for llama + assert os.path.isdir( + checkpoint + ), "You are in mem_efficient_load mode. \n Please set --load_quant the path to the folder containing all checkpoint files." + model = mem_efficient_load_checkpoint( + model, + checkpoint, + ) + else: + ckpt_version_check(checkpoint) + pbar = tqdm(range(1)) + pbar.set_description("Loading checkpoint") + for i in pbar: + if checkpoint.endswith(".safetensors"): + from safetensors.torch import load_file as safe_load + + model.load_state_dict(safe_load(checkpoint)) + else: + model.load_state_dict(torch.load(checkpoint)) + + return model.to(device) diff --git a/llm-awq/tinychat/utils/log_utils.py b/llm-awq/tinychat/utils/log_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ddb1d22da5b9585a0d0b8f34011689f0ceb5a5b8 --- /dev/null +++ b/llm-awq/tinychat/utils/log_utils.py @@ -0,0 +1,150 @@ +# Modified from https://github.com/haotian-liu/LLaVA +# Copyright 2023 Haotian Liu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import logging.handlers +import os +import sys + +import requests + +LOGDIR = "." + +server_error_msg = ( + "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**" +) +moderation_msg = ( + "YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES. PLEASE TRY AGAIN." +) + +handler = None + + +def build_logger(logger_name, logger_filename): + global handler + + formatter = logging.Formatter( + fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + # Set the format of root handlers + if not logging.getLogger().handlers: + logging.basicConfig(level=logging.INFO) + logging.getLogger().handlers[0].setFormatter(formatter) + + # Redirect stdout and stderr to loggers + stdout_logger = logging.getLogger("stdout") + stdout_logger.setLevel(logging.INFO) + sl = StreamToLogger(stdout_logger, logging.INFO) + sys.stdout = sl + + stderr_logger = logging.getLogger("stderr") + stderr_logger.setLevel(logging.ERROR) + sl = StreamToLogger(stderr_logger, logging.ERROR) + sys.stderr = sl + + # Get logger + logger = logging.getLogger(logger_name) + logger.setLevel(logging.INFO) + + # Add a file handler for all loggers + if handler is None: + os.makedirs(LOGDIR, exist_ok=True) + filename = os.path.join(LOGDIR, logger_filename) + handler = logging.handlers.TimedRotatingFileHandler( + filename, when="D", utc=True, encoding="UTF-8" + ) + handler.setFormatter(formatter) + + for name, item in logging.root.manager.loggerDict.items(): + if isinstance(item, logging.Logger): + item.addHandler(handler) + + return logger + + +class StreamToLogger(object): + """ + Fake file-like stream object that redirects writes to a logger instance. + """ + + def __init__(self, logger, log_level=logging.INFO): + self.terminal = sys.stdout + self.logger = logger + self.log_level = log_level + self.linebuf = "" + + def __getattr__(self, attr): + return getattr(self.terminal, attr) + + def write(self, buf): + temp_linebuf = self.linebuf + buf + self.linebuf = "" + for line in temp_linebuf.splitlines(True): + # From the io.TextIOWrapper docs: + # On output, if newline is None, any '\n' characters written + # are translated to the system default line separator. + # By default sys.stdout.write() expects '\n' newlines and then + # translates them so this is still cross platform. + if line[-1] == "\n": + self.logger.log(self.log_level, line.rstrip()) + else: + self.linebuf += line + + def flush(self): + if self.linebuf != "": + self.logger.log(self.log_level, self.linebuf.rstrip()) + self.linebuf = "" + + +def disable_torch_init(): + """ + Disable the redundant torch default initialization to accelerate model creation. + """ + import torch + + setattr(torch.nn.Linear, "reset_parameters", lambda self: None) + setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None) + + +def violates_moderation(text): + """ + Check whether the text violates OpenAI moderation API. + """ + url = "https://api.openai.com/v1/moderations" + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer " + os.environ["OPENAI_API_KEY"], + } + text = text.replace("\n", "") + data = "{" + '"input": ' + f'"{text}"' + "}" + data = data.encode("utf-8") + try: + ret = requests.post(url, headers=headers, data=data, timeout=5) + flagged = ret.json()["results"][0]["flagged"] + except requests.exceptions.RequestException as e: + flagged = False + except KeyError as e: + flagged = False + + return flagged + + +def pretty_print_semaphore(semaphore): + if semaphore is None: + return "None" + return f"Semaphore(value={semaphore._value}, locked={semaphore.locked()})" diff --git a/llm-awq/tinychat/utils/tune.py b/llm-awq/tinychat/utils/tune.py new file mode 100644 index 0000000000000000000000000000000000000000..40b69ce064859f31a744658c2fd1a472b1623f93 --- /dev/null +++ b/llm-awq/tinychat/utils/tune.py @@ -0,0 +1,81 @@ +import numpy as np +import time +import torch +from awq.quantize.qmodule import WQLinear + + +__all__ = ["device_warmup", "tune_all_wqlinears"] + + +def device_warmup(device: str): + warm_up = torch.randn((8192, 8192)).to(device) + for i in range(100): + torch.mm(warm_up, warm_up) + + +def tune_llava_patch_embedding(vision_tower, device): + # run the llava_patch_embedding layer to pre-tune the kernel configuration + # Without this pre-tuning, the embedding layer can cause significant slowdown due to cuDNN tuning. + device = vision_tower.device + if "intern" not in vision_tower.__class__.__name__.lower(): + patch_embedding = ( + vision_tower.vision_tower.vision_model.embeddings.patch_embedding + ) + else: + patch_embedding = vision_tower.vision_tower.embeddings.patch_embedding + patch_embedding = patch_embedding.to(device) + image = ( + torch.randn((1, patch_embedding.in_channels, 336, 336)) + .to(device) + .to(patch_embedding.weight.dtype) + ) + for i in range(100): + patch_embedding(image) + + +def _time_module(module, inputs, measure_iters=1000): + time_lis = [] + # Warmup + for i in range(measure_iters): + module(inputs) + for i in range(measure_iters): + torch.cuda.synchronize() + st = time.time() + module(inputs) + torch.cuda.synchronize() + ed = time.time() + time_lis.append((ed - st)) + return np.median(time_lis) + + +def tune_wqlinear(module: WQLinear, measure_iters: int = 1000): + device_warmup(str(module.scales.device)) + inputs = torch.randn( + 1, module.in_features, device=module.scales.device, dtype=module.scales.dtype + ) + best_split_k_iter = None + best_latency = None + for split_k_iters in [1, 2, 4, 8, 16, 32]: + module.split_k_iters = split_k_iters + cur_latency = _time_module(module, inputs, measure_iters) + if best_split_k_iter is None or best_latency >= cur_latency: + best_split_k_iter = split_k_iters + best_latency = cur_latency + module.split_k_iters = best_split_k_iter + return best_split_k_iter + + +def tune_all_wqlinears(model, measure_iters: int = 1000): + tuned_results = dict() + for name, module in model.named_modules(): + if isinstance(module, WQLinear): + ic, oc = module.in_features, module.out_features + if (ic, oc) not in tuned_results: + print(f"Tuning {(ic, oc)}...") + split_k_iters = tune_wqlinear(module) + tuned_results[(ic, oc)] = split_k_iters + # write configs to model + for name, module in model.named_modules(): + if isinstance(module, WQLinear): + ic, oc = module.in_features, module.out_features + module.split_k_iters = tuned_results[(ic, oc)] diff --git a/lm-evaluation-harness/.coveragerc b/lm-evaluation-harness/.coveragerc new file mode 100644 index 0000000000000000000000000000000000000000..1248476304d3d43662439148724428792f150585 --- /dev/null +++ b/lm-evaluation-harness/.coveragerc @@ -0,0 +1,28 @@ +[run] + +# tasks that aren't wired up. +omit = + lm_eval/tasks/quac.py + lm_eval/tasks/storycloze.py + lm_eval/tasks/cbt.py + lm_eval/tasks/sat.py + lm_eval/tasks/triviaqa.py + lm_eval/tasks/naturalqs.py + lm_eval/models/dummy.py + +[report] +exclude_lines = + # Skip any pass lines such as may be used for @abstractmethod + pass + + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + return NotImplemented diff --git a/lm-evaluation-harness/.flake8 b/lm-evaluation-harness/.flake8 new file mode 100644 index 0000000000000000000000000000000000000000..73f6455d132003fce0034f41d72eeb901b68f039 --- /dev/null +++ b/lm-evaluation-harness/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = E203, E266, E501, W503, F403, F401, C901 +max-line-length = 127 +max-complexity = 10 +select = B,C,E,F,W,T4,B9 diff --git a/lm-evaluation-harness/.github/workflows/new_tasks.yml b/lm-evaluation-harness/.github/workflows/new_tasks.yml new file mode 100644 index 0000000000000000000000000000000000000000..79567bfb8d4ef45ad1a4ef9aca62b8e24ca8080a --- /dev/null +++ b/lm-evaluation-harness/.github/workflows/new_tasks.yml @@ -0,0 +1,71 @@ +name: Tasks Modified + +on: + push: + branches: + - 'main' + pull_request: + branches: + - 'main' + workflow_dispatch: +# comment/edit out the above to stop/change the triggers +jobs: + changed_files: + runs-on: ubuntu-latest # windows-latest || macos-latest + timeout-minutes: 120 + name: Scan for changed tasks + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 2 # OR "2" -> To retrieve the preceding commit. + + # Uses the tj-actions/changed-files action to check for changes. + # The `files_yaml` input optionally takes a yaml string to specify filters, + # and prepends the filter name to the standard output names. + - name: Check task folders + id: changed-tasks + uses: tj-actions/changed-files@v46.0.5 + with: + # tasks checks the tasks folder and api checks the api folder for changes + files_yaml: | + tasks: + - lm_eval/tasks/** + api: + - lm_eval/api/** + write_output_files: true + + # The next step is optional; the files are written to the workspace by default (above). + # so it's just for debugging + - name: Run Tests + if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true' + run: | + echo .github/outputs/tasks_all_changed_and_modified_files.txt >> 'GITHUB_ENV' + echo "One or more test file(s) has changed." + echo "List of all the files that have changed: ${{ steps.changed-tasks.outputs.tasks_all_modified_files }}" + + - name: Set up Python 3.9 + if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true' + uses: actions/setup-python@v5 + with: + python-version: 3.9 + cache: 'pip' + cache-dependency-path: setup.py + - name: Install dependencies + if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true' + run: | + python -m pip install --upgrade pip + pip install -e '.[dev,ifeval]' --extra-index-url https://download.pytorch.org/whl/cpu + # Install optional git dependencies + # pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt + # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Test with pytest + # if new tasks are added, run tests on them + if: steps.changed-tasks.outputs.tasks_any_modified == 'true' + run: python -m pytest tests/test_tasks.py -s -vv + # if api is modified, run tests on it + - name: Test more tasks with pytest + env: + API: true + if: steps.changed-tasks.outputs.api_any_modified == 'true' + run: python -m pytest tests/test_tasks.py -s -vv diff --git a/lm-evaluation-harness/.github/workflows/publish.yml b/lm-evaluation-harness/.github/workflows/publish.yml new file mode 100644 index 0000000000000000000000000000000000000000..a053961669ed99ce82bf3546ad93609a1b94283f --- /dev/null +++ b/lm-evaluation-harness/.github/workflows/publish.yml @@ -0,0 +1,97 @@ +name: Publish Python distribution to PyPI + +on: + push: + tags: + - '*' + +jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Check version consistency + run: | + # Extract version from pyproject.toml + PYPROJECT_VERSION=$(grep 'version = ' pyproject.toml | head -1 | cut -d'"' -f2) + + # Extract version from __init__.py + INIT_VERSION=$(grep '__version__ = ' lm_eval/__init__.py | head -1 | cut -d'"' -f2) + + echo "Version in pyproject.toml: $PYPROJECT_VERSION" + echo "Version in __init__.py: $INIT_VERSION" + + # Check if versions match + if [ "$PYPROJECT_VERSION" != "$INIT_VERSION" ]; then + echo "Error: Version mismatch between pyproject.toml ($PYPROJECT_VERSION) and __init__.py ($INIT_VERSION)" + exit 1 + fi + + echo "Version check passed: $PYPROJECT_VERSION" + + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python distribution to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/lm_eval + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + publish-to-testpypi: + name: Publish Python distribution to TestPyPI + needs: + - build + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/p/lm_eval + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ diff --git a/lm-evaluation-harness/.github/workflows/unit_tests.yml b/lm-evaluation-harness/.github/workflows/unit_tests.yml new file mode 100644 index 0000000000000000000000000000000000000000..b9a448642dbb5ef70339a324a6b58641942b2506 --- /dev/null +++ b/lm-evaluation-harness/.github/workflows/unit_tests.yml @@ -0,0 +1,114 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python +# just comment out unwanted steps to turn off the test. +name: Unit Tests + +on: + push: + branches: + - 'main' + pull_request: + branches: + - 'main' + workflow_dispatch: +# Jobs run concurrently and steps run sequentially within a job. +# jobs: linter and cpu_tests. Add more jobs/steps as required. +jobs: + linter: + name: Linters + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: 3.9 + cache: pip + cache-dependency-path: pyproject.toml + - name: Pre-Commit + env: + SKIP: "no-commit-to-branch,mypy" + uses: pre-commit/action@v3.0.1 + # Job 2 + testcpu: + name: CPU Tests + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + python-version: ["3.9", "3.10", "3.11"] + timeout-minutes: 30 + steps: + - name: Checkout Code + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml + + # Cache HuggingFace cache directory for CPU tests + - name: Cache HuggingFace cache (CPU tests) + uses: actions/cache@v3 + id: cache-hf-cpu + with: + path: ~/.cache/huggingface + key: ${{ runner.os }}-hf-cache-cpu + restore-keys: | + ${{ runner.os }}-hf-cache-cpu + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e '.[dev]' --extra-index-url https://download.pytorch.org/whl/cpu + pip install hf_xet + + - name: Test with pytest + run: python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_neuralmagic.py --ignore=tests/models/test_openvino.py --ignore=tests/models/test_hf_steered.py + continue-on-error: true # Continue workflow even if tests fail + + # Save test artifacts + - name: Archive test artifacts + uses: actions/upload-artifact@v4 + with: + name: output_testcpu${{ matrix.python-version }} + path: | + test_logs/* + +# testmodels: +# name: External LM Tests +# runs-on: ubuntu-latest +# timeout-minutes: 30 +# steps: +# - name: Checkout Code +# uses: actions/checkout@v4 +# - name: Set up Python 3.9 +# uses: actions/setup-python@v5 +# with: +# python-version: 3.9 +# cache: pip +# cache-dependency-path: pyproject.toml +# +# # Cache HuggingFace cache directory for External LM tests +# - name: Cache HuggingFace cache (External LM tests) +# uses: actions/cache@v3 +# id: cache-hf-lm +# with: +# path: ~/.cache/huggingface +# key: ${{ runner.os }}-hf-cache-external-lm +# restore-keys: | +# ${{ runner.os }}-hf-cache-external-lm +# +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip +# pip install -e '.[dev,optimum,deepsparse,sparseml,api]' --extra-index-url https://download.pytorch.org/whl/cpu +# pip install -U transformers peft accelerate +# +# - name: Test with pytest +# run: python -m pytest tests/models --showlocals -s -vv +# continue-on-error: true # Continue workflow even if tests fail diff --git a/lm-evaluation-harness/CITATION.bib b/lm-evaluation-harness/CITATION.bib new file mode 100644 index 0000000000000000000000000000000000000000..4ec33f139693aad74d2cb89c5edb2a578a315dd2 --- /dev/null +++ b/lm-evaluation-harness/CITATION.bib @@ -0,0 +1,10 @@ +@misc{eval-harness, + author = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy}, + title = {A framework for few-shot language model evaluation}, + month = 12, + year = 2023, + publisher = {Zenodo}, + version = {v0.4.0}, + doi = {10.5281/zenodo.10256836}, + url = {https://zenodo.org/records/10256836} +} diff --git a/lm-evaluation-harness/CODEOWNERS b/lm-evaluation-harness/CODEOWNERS new file mode 100644 index 0000000000000000000000000000000000000000..32f61c33f06047800239d50416a2f86a8f63c6bd --- /dev/null +++ b/lm-evaluation-harness/CODEOWNERS @@ -0,0 +1 @@ +* @baberabb @stellaathena diff --git a/lm-evaluation-harness/LICENSE.md b/lm-evaluation-harness/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..12e6063183935e876e232db276568baf4954b492 --- /dev/null +++ b/lm-evaluation-harness/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 EleutherAI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lm-evaluation-harness/MANIFEST.in b/lm-evaluation-harness/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..93f181def4d46b013259fae732c03ce172127da8 --- /dev/null +++ b/lm-evaluation-harness/MANIFEST.in @@ -0,0 +1 @@ +recursive-include tests diff --git a/lm-evaluation-harness/README.md b/lm-evaluation-harness/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f325ae478dad8bba17e8a35d0ec940834e545f63 --- /dev/null +++ b/lm-evaluation-harness/README.md @@ -0,0 +1,625 @@ +# Language Model Evaluation Harness + +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10256836.svg)](https://doi.org/10.5281/zenodo.10256836) + +--- + +## Latest News ๐Ÿ“ฃ + +- [2025/03] Added support for steering HF models! +- [2025/02] Added [SGLang](https://docs.sglang.ai/) support! +- [2024/09] We are prototyping allowing users of LM Evaluation Harness to create and evaluate on text+image multimodal input, text output tasks, and have just added the `hf-multimodal` and `vllm-vlm` model types and `mmmu` task as a prototype feature. We welcome users to try out this in-progress feature and stress-test it for themselves, and suggest they check out [`lmms-eval`](https://github.com/EvolvingLMMs-Lab/lmms-eval), a wonderful project originally forking off of the lm-evaluation-harness, for a broader range of multimodal tasks, models, and features. +- [2024/07] [API model](docs/API_guide.md) support has been updated and refactored, introducing support for batched and async requests, and making it significantly easier to customize and use for your own purposes. **To run Llama 405B, we recommend using VLLM's OpenAI-compliant API to host the model, and use the `local-completions` model type to evaluate the model.** +- [2024/07] New Open LLM Leaderboard tasks have been added ! You can find them under the [leaderboard](lm_eval/tasks/leaderboard/README.md) task group. + +--- + +## Announcement + +**A new v0.4.0 release of lm-evaluation-harness is available** ! + +New updates and features include: + +- **New Open LLM Leaderboard tasks have been added ! You can find them under the [leaderboard](lm_eval/tasks/leaderboard/README.md) task group.** +- Internal refactoring +- Config-based task creation and configuration +- Easier import and sharing of externally-defined task config YAMLs +- Support for Jinja2 prompt design, easy modification of prompts + prompt imports from Promptsource +- More advanced configuration options, including output post-processing, answer extraction, and multiple LM generations per document, configurable fewshot settings, and more +- Speedups and new modeling libraries supported, including: faster data-parallel HF model usage, vLLM support, MPS support with HuggingFace, and more +- Logging and usability changes +- New tasks including CoT BIG-Bench-Hard, Belebele, user-defined task groupings, and more + +Please see our updated documentation pages in `docs/` for more details. + +Development will be continuing on the `main` branch, and we encourage you to give us feedback on what features are desired and how to improve the library further, or ask questions, either in issues or PRs on GitHub, or in the [EleutherAI discord](https://discord.gg/eleutherai)! + +--- + +## Overview + +This project provides a unified framework to test generative language models on a large number of different evaluation tasks. + +**Features:** + +- Over 60 standard academic benchmarks for LLMs, with hundreds of subtasks and variants implemented. +- Support for models loaded via [transformers](https://github.com/huggingface/transformers/) (including quantization via [GPTQModel](https://github.com/ModelCloud/GPTQModel) and [AutoGPTQ](https://github.com/PanQiWei/AutoGPTQ)), [GPT-NeoX](https://github.com/EleutherAI/gpt-neox), and [Megatron-DeepSpeed](https://github.com/microsoft/Megatron-DeepSpeed/), with a flexible tokenization-agnostic interface. +- Support for fast and memory-efficient inference with [vLLM](https://github.com/vllm-project/vllm). +- Support for commercial APIs including [OpenAI](https://openai.com), and [TextSynth](https://textsynth.com/). +- Support for evaluation on adapters (e.g. LoRA) supported in [HuggingFace's PEFT library](https://github.com/huggingface/peft). +- Support for local models and benchmarks. +- Evaluation with publicly available prompts ensures reproducibility and comparability between papers. +- Easy support for custom prompts and evaluation metrics. + +The Language Model Evaluation Harness is the backend for ๐Ÿค— Hugging Face's popular [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard), has been used in [hundreds of papers](https://scholar.google.com/scholar?oi=bibs&hl=en&authuser=2&cites=15052937328817631261,4097184744846514103,1520777361382155671,17476825572045927382,18443729326628441434,14801318227356878622,7890865700763267262,12854182577605049984,15641002901115500560,5104500764547628290), and is used internally by dozens of organizations including NVIDIA, Cohere, BigScience, BigCode, Nous Research, and Mosaic ML. + +## Install + +To install the `lm-eval` package from the github repository, run: + +```bash +git clone --depth 1 https://github.com/EleutherAI/lm-evaluation-harness +cd lm-evaluation-harness +pip install -e . +``` + +We also provide a number of optional dependencies for extended functionality. A detailed table is available at the end of this document. + +## Basic Usage + +### User Guide + +A user guide detailing the full list of supported arguments is provided [here](./docs/interface.md), and on the terminal by calling `lm_eval -h`. Alternatively, you can use `lm-eval` instead of `lm_eval`. + +A list of supported tasks (or groupings of tasks) can be viewed with `lm-eval --tasks list`. Task descriptions and links to corresponding subfolders are provided [here](./lm_eval/tasks/README.md). + +### Hugging Face `transformers` + +To evaluate a model hosted on the [HuggingFace Hub](https://huggingface.co/models) (e.g. GPT-J-6B) on `hellaswag` you can use the following command (this assumes you are using a CUDA-compatible GPU): + +```bash +lm_eval --model hf \ + --model_args pretrained=EleutherAI/gpt-j-6B \ + --tasks hellaswag \ + --device cuda:0 \ + --batch_size 8 +``` + +Additional arguments can be provided to the model constructor using the `--model_args` flag. Most notably, this supports the common practice of using the `revisions` feature on the Hub to store partially trained checkpoints, or to specify the datatype for running a model: + +```bash +lm_eval --model hf \ + --model_args pretrained=EleutherAI/pythia-160m,revision=step100000,dtype="float" \ + --tasks lambada_openai,hellaswag \ + --device cuda:0 \ + --batch_size 8 +``` + +Models that are loaded via both `transformers.AutoModelForCausalLM` (autoregressive, decoder-only GPT style models) and `transformers.AutoModelForSeq2SeqLM` (such as encoder-decoder models like T5) in Huggingface are supported. + +Batch size selection can be automated by setting the ```--batch_size``` flag to ```auto```. This will perform automatic detection of the largest batch size that will fit on your device. On tasks where there is a large difference between the longest and shortest example, it can be helpful to periodically recompute the largest batch size, to gain a further speedup. To do this, append ```:N``` to above flag to automatically recompute the largest batch size ```N``` times. For example, to recompute the batch size 4 times, the command would be: + +```bash +lm_eval --model hf \ + --model_args pretrained=EleutherAI/pythia-160m,revision=step100000,dtype="float" \ + --tasks lambada_openai,hellaswag \ + --device cuda:0 \ + --batch_size auto:4 +``` + +> [!Note] +> Just like you can provide a local path to `transformers.AutoModel`, you can also provide a local path to `lm_eval` via `--model_args pretrained=/path/to/model` + +#### Multi-GPU Evaluation with Hugging Face `accelerate` + +We support three main ways of using Hugging Face's [accelerate ๐Ÿš€](https://github.com/huggingface/accelerate) library for multi-GPU evaluation. + +To perform *data-parallel evaluation* (where each GPU loads a **separate full copy** of the model), we leverage the `accelerate` launcher as follows: + +```bash +accelerate launch -m lm_eval --model hf \ + --tasks lambada_openai,arc_easy \ + --batch_size 16 +``` + +(or via `accelerate launch --no-python lm_eval`). + +For cases where your model can fit on a single GPU, this allows you to evaluate on K GPUs K times faster than on one. + +**WARNING**: This setup does not work with FSDP model sharding, so in `accelerate config` FSDP must be disabled, or the NO_SHARD FSDP option must be used. + +The second way of using `accelerate` for multi-GPU evaluation is when your model is *too large to fit on a single GPU.* + +In this setting, run the library *outside the `accelerate` launcher*, but passing `parallelize=True` to `--model_args` as follows: + +```bash +lm_eval --model hf \ + --tasks lambada_openai,arc_easy \ + --model_args parallelize=True \ + --batch_size 16 +``` + +This means that your model's weights will be split across all available GPUs. + +For more advanced users or even larger models, we allow for the following arguments when `parallelize=True` as well: + +- `device_map_option`: How to split model weights across available GPUs. defaults to "auto". +- `max_memory_per_gpu`: the max GPU memory to use per GPU in loading the model. +- `max_cpu_memory`: the max amount of CPU memory to use when offloading the model weights to RAM. +- `offload_folder`: a folder where model weights will be offloaded to disk if needed. + +The third option is to use both at the same time. This will allow you to take advantage of both data parallelism and model sharding, and is especially useful for models that are too large to fit on a single GPU. + +```bash +accelerate launch --multi_gpu --num_processes {nb_of_copies_of_your_model} \ + -m lm_eval --model hf \ + --tasks lambada_openai,arc_easy \ + --model_args parallelize=True \ + --batch_size 16 +``` + +To learn more about model parallelism and how to use it with the `accelerate` library, see the [accelerate documentation](https://huggingface.co/docs/transformers/v4.15.0/en/parallelism) + +**Warning: We do not natively support multi-node evaluation using the `hf` model type! Please reference [our GPT-NeoX library integration](https://github.com/EleutherAI/gpt-neox/blob/main/eval.py) for an example of code in which a custom multi-machine evaluation script is written.** + +**Note: we do not currently support multi-node evaluations natively, and advise using either an externally hosted server to run inference requests against, or creating a custom integration with your distributed framework [as is done for the GPT-NeoX library](https://github.com/EleutherAI/gpt-neox/blob/main/eval_tasks/eval_adapter.py).** + +### Steered Hugging Face `transformers` models + +To evaluate a Hugging Face `transformers` model with steering vectors applied, specify the model type as `steered` and provide the path to either a PyTorch file containing pre-defined steering vectors, or a CSV file that specifies how to derive steering vectors from pretrained `sparsify` or `sae_lens` models (you will need to install the corresponding optional dependency for this method). + +Specify pre-defined steering vectors: + +```python +import torch + +steer_config = { + "layers.3": { + "steering_vector": torch.randn(1, 768), + "bias": torch.randn(1, 768), + "steering_coefficient": 1, + "action": "add" + }, +} +torch.save(steer_config, "steer_config.pt") +``` + +Specify derived steering vectors: + +```python +import pandas as pd + +pd.DataFrame({ + "loader": ["sparsify"], + "action": ["add"], + "sparse_model": ["EleutherAI/sae-pythia-70m-32k"], + "hookpoint": ["layers.3"], + "feature_index": [30], + "steering_coefficient": [10.0], +}).to_csv("steer_config.csv", index=False) +``` + +Run the evaluation harness with steering vectors applied: + +```bash +lm_eval --model steered \ + --model_args pretrained=EleutherAI/pythia-160m,steer_path=steer_config.pt \ + --tasks lambada_openai,hellaswag \ + --device cuda:0 \ + --batch_size 8 +``` + +### NVIDIA `nemo` models + +[NVIDIA NeMo Framework](https://github.com/NVIDIA/NeMo) is a generative AI framework built for researchers and pytorch developers working on language models. + +To evaluate a `nemo` model, start by installing NeMo following [the documentation](https://github.com/NVIDIA/NeMo?tab=readme-ov-file#installation). We highly recommended to use the NVIDIA PyTorch or NeMo container, especially if having issues installing Apex or any other dependencies (see [latest released containers](https://github.com/NVIDIA/NeMo/releases)). Please also install the lm evaluation harness library following the instructions in [the Install section](https://github.com/EleutherAI/lm-evaluation-harness/tree/main?tab=readme-ov-file#install). + +NeMo models can be obtained through [NVIDIA NGC Catalog](https://catalog.ngc.nvidia.com/models) or in [NVIDIA's Hugging Face page](https://huggingface.co/nvidia). In [NVIDIA NeMo Framework](https://github.com/NVIDIA/NeMo/tree/main/scripts/nlp_language_modeling) there are conversion scripts to convert the `hf` checkpoints of popular models like llama, falcon, mixtral or mpt to `nemo`. + +Run a `nemo` model on one GPU: + +```bash +lm_eval --model nemo_lm \ + --model_args path= \ + --tasks hellaswag \ + --batch_size 32 +``` + +It is recommended to unpack the `nemo` model to avoid the unpacking inside the docker container - it may overflow disk space. For that you can run: + +```bash +mkdir MY_MODEL +tar -xvf MY_MODEL.nemo -c MY_MODEL +``` + +#### Multi-GPU evaluation with NVIDIA `nemo` models + +By default, only one GPU is used. But we do support either data replication or tensor/pipeline parallelism during evaluation, on one node. + +1) To enable data replication, set the `model_args` of `devices` to the number of data replicas to run. For example, the command to run 8 data replicas over 8 GPUs is: + +```bash +torchrun --nproc-per-node=8 --no-python lm_eval \ + --model nemo_lm \ + --model_args path=,devices=8 \ + --tasks hellaswag \ + --batch_size 32 +``` + +1) To enable tensor and/or pipeline parallelism, set the `model_args` of `tensor_model_parallel_size` and/or `pipeline_model_parallel_size`. In addition, you also have to set up `devices` to be equal to the product of `tensor_model_parallel_size` and/or `pipeline_model_parallel_size`. For example, the command to use one node of 4 GPUs with tensor parallelism of 2 and pipeline parallelism of 2 is: + +```bash +torchrun --nproc-per-node=4 --no-python lm_eval \ + --model nemo_lm \ + --model_args path=,devices=4,tensor_model_parallel_size=2,pipeline_model_parallel_size=2 \ + --tasks hellaswag \ + --batch_size 32 +``` + +Note that it is recommended to substitute the `python` command by `torchrun --nproc-per-node= --no-python` to facilitate loading the model into the GPUs. This is especially important for large checkpoints loaded into multiple GPUs. + +Not supported yet: multi-node evaluation and combinations of data replication with tensor or pipeline parallelism. + +#### Multi-GPU evaluation with OpenVINO models + +Pipeline parallelism during evaluation is supported with OpenVINO models + +To enable pipeline parallelism, set the `model_args` of `pipeline_parallel`. In addition, you also have to set up `device` to value `HETERO:,` for example `HETERO:GPU.1,GPU.0` For example, the command to use pipeline parallelism of 2 is: + +```bash +lm_eval --model openvino \ + --tasks wikitext \ + --model_args pretrained=,pipeline_parallel=True \ + --device HETERO:GPU.1,GPU.0 +``` + +### Tensor + Data Parallel and Optimized Inference with `vLLM` + +We also support vLLM for faster inference on [supported model types](https://docs.vllm.ai/en/latest/models/supported_models.html), especially faster when splitting a model across multiple GPUs. For single-GPU or multi-GPU โ€” tensor parallel, data parallel, or a combination of both โ€” inference, for example: + +```bash +lm_eval --model vllm \ + --model_args pretrained={model_name},tensor_parallel_size={GPUs_per_model},dtype=auto,gpu_memory_utilization=0.8,data_parallel_size={model_replicas} \ + --tasks lambada_openai \ + --batch_size auto +``` + +To use vllm, do `pip install lm_eval[vllm]`. For a full list of supported vLLM configurations, please reference our [vLLM integration](https://github.com/EleutherAI/lm-evaluation-harness/blob/e74ec966556253fbe3d8ecba9de675c77c075bce/lm_eval/models/vllm_causallms.py) and the vLLM documentation. + +vLLM occasionally differs in output from Huggingface. We treat Huggingface as the reference implementation, and provide a [script](./scripts/model_comparator.py) for checking the validity of vllm results against HF. + +> [!Tip] +> For fastest performance, we recommend using `--batch_size auto` for vLLM whenever possible, to leverage its continuous batching functionality! + +> [!Tip] +> Passing `max_model_len=4096` or some other reasonable default to vLLM through model args may cause speedups or prevent out-of-memory errors when trying to use auto batch size, such as for Mistral-7B-v0.1 which defaults to a maximum length of 32k. + +### Tensor + Data Parallel and Fast Offline Batching Inference with `SGLang` + +We support SGLang for efficient offline batch inference. Its **[Fast Backend Runtime](https://docs.sglang.ai/index.html)** delivers high performance through optimized memory management and parallel processing techniques. Key features include tensor parallelism, continuous batching, and support for various quantization methods (FP8/INT4/AWQ/GPTQ). + +To use SGLang as the evaluation backend, please **install it in advance** via SGLang documents [here](https://docs.sglang.ai/start/install.html#install-sglang). + +> [!Tip] +> Due to the installing method of [`Flashinfer`](https://docs.flashinfer.ai/)-- a fast attention kernel library, we don't include the dependencies of `SGLang` within [pyproject.toml](pyproject.toml). Note that the `Flashinfer` also has some requirements on `torch` version. + +SGLang's server arguments are slightly different from other backends, see [here](https://docs.sglang.ai/backend/server_arguments.html) for more information. We provide an example of the usage here: + +```bash +lm_eval --model sglang \ + --model_args pretrained={model_name},dp_size={data_parallel_size},tp_size={tensor_parallel_size},dtype=auto \ + --tasks gsm8k_cot \ + --batch_size auto +``` + +> [!Tip] +> When encountering out of memory (OOM) errors (especially for multiple-choice tasks), try these solutions: +> +> 1. Use a manual `batch_size`, rather than `auto`. +> 2. Lower KV cache pool memory usage by adjusting `mem_fraction_static` - Add to your model arguments for example `--model_args pretrained=...,mem_fraction_static=0.7`. +> 3. Increase tensor parallel size `tp_size` (if using multiple GPUs). + +### Model APIs and Inference Servers + +Our library also supports the evaluation of models served via several commercial APIs, and we hope to implement support for the most commonly used performant local/self-hosted inference servers. + +To call a hosted model, use: + +```bash +export OPENAI_API_KEY=YOUR_KEY_HERE +lm_eval --model openai-completions \ + --model_args model=davinci-002 \ + --tasks lambada_openai,hellaswag +``` + +We also support using your own local inference server with servers that mirror the OpenAI Completions and ChatCompletions APIs. + +```bash +lm_eval --model local-completions --tasks gsm8k --model_args model=facebook/opt-125m,base_url=http://{yourip}:8000/v1/completions,num_concurrent=1,max_retries=3,tokenized_requests=False,batch_size=16 +``` + +Note that for externally hosted models, configs such as `--device` which relate to where to place a local model should not be used and do not function. Just like you can use `--model_args` to pass arbitrary arguments to the model constructor for local models, you can use it to pass arbitrary arguments to the model API for hosted models. See the documentation of the hosting service for information on what arguments they support. + +| API or Inference Server | Implemented? | `--model ` name | Models supported: | Request Types: | +| --------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|-----------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------| +| OpenAI Completions | :heavy_check_mark: | `openai-completions`, `local-completions` | All OpenAI Completions API models | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| OpenAI ChatCompletions | :heavy_check_mark: | `openai-chat-completions`, `local-chat-completions` | [All ChatCompletions API models](https://platform.openai.com/docs/guides/gpt) | `generate_until` (no logprobs) | +| Anthropic | :heavy_check_mark: | `anthropic` | [Supported Anthropic Engines](https://docs.anthropic.com/claude/reference/selecting-a-model) | `generate_until` (no logprobs) | +| Anthropic Chat | :heavy_check_mark: | `anthropic-chat`, `anthropic-chat-completions` | [Supported Anthropic Engines](https://docs.anthropic.com/claude/docs/models-overview) | `generate_until` (no logprobs) | +| Textsynth | :heavy_check_mark: | `textsynth` | [All supported engines](https://textsynth.com/documentation.html#engines) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| Cohere | [:hourglass: - blocked on Cohere API bug](https://github.com/EleutherAI/lm-evaluation-harness/pull/395) | N/A | [All `cohere.generate()` engines](https://docs.cohere.com/docs/models) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| [Llama.cpp](https://github.com/ggerganov/llama.cpp) (via [llama-cpp-python](https://github.com/abetlen/llama-cpp-python)) | :heavy_check_mark: | `gguf`, `ggml` | [All models supported by llama.cpp](https://github.com/ggerganov/llama.cpp) | `generate_until`, `loglikelihood`, (perplexity evaluation not yet implemented) | +| vLLM | :heavy_check_mark: | `vllm` | [Most HF Causal Language Models](https://docs.vllm.ai/en/latest/models/supported_models.html) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| Mamba | :heavy_check_mark: | `mamba_ssm` | [Mamba architecture Language Models via the `mamba_ssm` package](https://huggingface.co/state-spaces) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| Huggingface Optimum (Causal LMs) | :heavy_check_mark: | `openvino` | Any decoder-only AutoModelForCausalLM converted with Huggingface Optimum into OpenVINOโ„ข Intermediate Representation (IR) format | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| Huggingface Optimum-intel IPEX (Causal LMs) | :heavy_check_mark: | `ipex` | Any decoder-only AutoModelForCausalLM | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| Neuron via AWS Inf2 (Causal LMs) | :heavy_check_mark: | `neuronx` | Any decoder-only AutoModelForCausalLM supported to run on [huggingface-ami image for inferentia2](https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| [Neural Magic DeepSparse](https://github.com/neuralmagic/deepsparse) | :heavy_check_mark: | `deepsparse` | Any LM from [SparseZoo](https://sparsezoo.neuralmagic.com/) or on [HF Hub with the "deepsparse" tag](https://huggingface.co/models?other=deepsparse) | `generate_until`, `loglikelihood` | +| [Neural Magic SparseML](https://github.com/neuralmagic/sparseml) | :heavy_check_mark: | `sparseml` | Any decoder-only AutoModelForCausalLM from [SparseZoo](https://sparsezoo.neuralmagic.com/) or on [HF Hub](https://huggingface.co/neuralmagic). Especially useful for models with quantization like [`zoo:llama2-7b-gsm8k_llama2_pretrain-pruned60_quantized`](https://sparsezoo.neuralmagic.com/models/llama2-7b-gsm8k_llama2_pretrain-pruned60_quantized) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| NVIDIA NeMo | :heavy_check_mark: | `nemo_lm` | [All supported models](https://docs.nvidia.com/nemo-framework/user-guide/24.09/nemotoolkit/core/core.html#nemo-models) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| Watsonx.ai | :heavy_check_mark: | `watsonx_llm` | [Supported Watsonx.ai Engines](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx) | `generate_until` `loglikelihood` | +| [Your local inference server!](docs/API_guide.md) | :heavy_check_mark: | `local-completions` or `local-chat-completions` | Support for OpenAI API-compatible servers, with easy customization for other APIs. | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | + +Models which do not supply logits or logprobs can be used with tasks of type `generate_until` only, while local models, or APIs that supply logprobs/logits of their prompts, can be run on all task types: `generate_until`, `loglikelihood`, `loglikelihood_rolling`, and `multiple_choice`. + +For more information on the different task `output_types` and model request types, see [our documentation](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/model_guide.md#interface). + +> [!Note] +> For best performance with closed chat model APIs such as Anthropic Claude 3 and GPT-4, we recommend carefully looking at a few sample outputs using `--limit 10` first to confirm answer extraction and scoring on generative tasks is performing as expected. providing `system=""` within `--model_args` for anthropic-chat-completions, to instruct the model what format to respond in, may be useful. + +### Other Frameworks + +A number of other libraries contain scripts for calling the eval harness through their library. These include [GPT-NeoX](https://github.com/EleutherAI/gpt-neox/blob/main/eval_tasks/eval_adapter.py), [Megatron-DeepSpeed](https://github.com/microsoft/Megatron-DeepSpeed/blob/main/examples/MoE/readme_evalharness.md), and [mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/blob/master/eval_harness.py). + +To create your own custom integration you can follow instructions from [this tutorial](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md#external-library-usage). + +### Additional Features + +> [!Note] +> For tasks unsuitable for direct evaluation โ€” either due risks associated with executing untrusted code or complexities in the evaluation process โ€” the `--predict_only` flag is available to obtain decoded generations for post-hoc evaluation. + +If you have a Metal compatible Mac, you can run the eval harness using the MPS back-end by replacing `--device cuda:0` with `--device mps` (requires PyTorch version 2.1 or higher). **Note that the PyTorch MPS backend is still in early stages of development, so correctness issues or unsupported operations may exist. If you observe oddities in model performance on the MPS back-end, we recommend first checking that a forward pass of your model on `--device cpu` and `--device mps` match.** + +> [!Note] +> You can inspect what the LM inputs look like by running the following command: +> +> ```bash +> python write_out.py \ +> --tasks \ +> --num_fewshot 5 \ +> --num_examples 10 \ +> --output_base_path /path/to/output/folder +> ``` +> +> This will write out one text file for each task. + +To verify the data integrity of the tasks you're performing in addition to running the tasks themselves, you can use the `--check_integrity` flag: + +```bash +lm_eval --model openai \ + --model_args engine=davinci-002 \ + --tasks lambada_openai,hellaswag \ + --check_integrity +``` + +## Advanced Usage Tips + +For models loaded with the HuggingFace `transformers` library, any arguments provided via `--model_args` get passed to the relevant constructor directly. This means that anything you can do with `AutoModel` can be done with our library. For example, you can pass a local path via `pretrained=` or use models finetuned with [PEFT](https://github.com/huggingface/peft) by taking the call you would run to evaluate the base model and add `,peft=PATH` to the `model_args` argument: + +```bash +lm_eval --model hf \ + --model_args pretrained=EleutherAI/gpt-j-6b,parallelize=True,load_in_4bit=True,peft=nomic-ai/gpt4all-j-lora \ + --tasks openbookqa,arc_easy,winogrande,hellaswag,arc_challenge,piqa,boolq \ + --device cuda:0 +``` + +Models provided as delta weights can be easily loaded using the Hugging Face transformers library. Within --model_args, set the delta argument to specify the delta weights, and use the pretrained argument to designate the relative base model to which they will be applied: + +```bash +lm_eval --model hf \ + --model_args pretrained=Ejafa/llama_7B,delta=lmsys/vicuna-7b-delta-v1.1 \ + --tasks hellaswag +``` + +GPTQ quantized models can be loaded using [GPTQModel](https://github.com/ModelCloud/GPTQModel) (faster) or [AutoGPTQ](https://github.com/PanQiWei/AutoGPTQ) + +GPTQModel: add `,gptqmodel=True` to `model_args` + +```bash +lm_eval --model hf \ + --model_args pretrained=model-name-or-path,gptqmodel=True \ + --tasks hellaswag +``` + +AutoGPTQ: add `,autogptq=True` to `model_args`: + +```bash +lm_eval --model hf \ + --model_args pretrained=model-name-or-path,autogptq=model.safetensors,gptq_use_triton=True \ + --tasks hellaswag +``` + +We support wildcards in task names, for example you can run all of the machine-translated lambada tasks via `--task lambada_openai_mt_*`. + +## Saving & Caching Results + +To save evaluation results provide an `--output_path`. We also support logging model responses with the `--log_samples` flag for post-hoc analysis. + +> [!TIP] +> Use `--use_cache ` to cache evaluation results and skip previously evaluated samples when resuming runs of the same (model, task) pairs. Note that caching is rank-dependent, so restart with the same GPU count if interrupted. You can also use --cache_requests to save dataset preprocessing steps for faster evaluation resumption. + +To push results and samples to the Hugging Face Hub, first ensure an access token with write access is set in the `HF_TOKEN` environment variable. Then, use the `--hf_hub_log_args` flag to specify the organization, repository name, repository visibility, and whether to push results and samples to the Hub - [example dataset on the HF Hub](https://huggingface.co/datasets/KonradSzafer/lm-eval-results-demo). For instance: + +```bash +lm_eval --model hf \ + --model_args pretrained=model-name-or-path,autogptq=model.safetensors,gptq_use_triton=True \ + --tasks hellaswag \ + --log_samples \ + --output_path results \ + --hf_hub_log_args hub_results_org=EleutherAI,hub_repo_name=lm-eval-results,push_results_to_hub=True,push_samples_to_hub=True,public_repo=False \ +``` + +This allows you to easily download the results and samples from the Hub, using: + +```python +from datasets import load_dataset + +load_dataset("EleutherAI/lm-eval-results-private", "hellaswag", "latest") +``` + +For a full list of supported arguments, check out the [interface](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md) guide in our documentation! + +## Visualizing Results + +You can seamlessly visualize and analyze the results of your evaluation harness runs using both Weights & Biases (W&B) and Zeno. + +### Zeno + +You can use [Zeno](https://zenoml.com) to visualize the results of your eval harness runs. + +First, head to [hub.zenoml.com](https://hub.zenoml.com) to create an account and get an API key [on your account page](https://hub.zenoml.com/account). +Add this key as an environment variable: + +```bash +export ZENO_API_KEY=[your api key] +``` + +You'll also need to install the `lm_eval[zeno]` package extra. + +To visualize the results, run the eval harness with the `log_samples` and `output_path` flags. +We expect `output_path` to contain multiple folders that represent individual model names. +You can thus run your evaluation on any number of tasks and models and upload all of the results as projects on Zeno. + +```bash +lm_eval \ + --model hf \ + --model_args pretrained=EleutherAI/gpt-j-6B \ + --tasks hellaswag \ + --device cuda:0 \ + --batch_size 8 \ + --log_samples \ + --output_path output/gpt-j-6B +``` + +Then, you can upload the resulting data using the `zeno_visualize` script: + +```bash +python scripts/zeno_visualize.py \ + --data_path output \ + --project_name "Eleuther Project" +``` + +This will use all subfolders in `data_path` as different models and upload all tasks within these model folders to Zeno. +If you run the eval harness on multiple tasks, the `project_name` will be used as a prefix and one project will be created per task. + +You can find an example of this workflow in [examples/visualize-zeno.ipynb](examples/visualize-zeno.ipynb). + +### Weights and Biases + +With the [Weights and Biases](https://wandb.ai/site) integration, you can now spend more time extracting deeper insights into your evaluation results. The integration is designed to streamline the process of logging and visualizing experiment results using the Weights & Biases (W&B) platform. + +The integration provide functionalities + +- to automatically log the evaluation results, +- log the samples as W&B Tables for easy visualization, +- log the `results.json` file as an artifact for version control, +- log the `_eval_samples.json` file if the samples are logged, +- generate a comprehensive report for analysis and visualization with all the important metric, +- log task and cli specific configs, +- and more out of the box like the command used to run the evaluation, GPU/CPU counts, timestamp, etc. + +First you'll need to install the lm_eval[wandb] package extra. Do `pip install lm_eval[wandb]`. + +Authenticate your machine with an your unique W&B token. Visit https://wandb.ai/authorize to get one. Do `wandb login` in your command line terminal. + +Run eval harness as usual with a `wandb_args` flag. Use this flag to provide arguments for initializing a wandb run ([wandb.init](https://docs.wandb.ai/ref/python/init)) as comma separated string arguments. + +```bash +lm_eval \ + --model hf \ + --model_args pretrained=microsoft/phi-2,trust_remote_code=True \ + --tasks hellaswag,mmlu_abstract_algebra \ + --device cuda:0 \ + --batch_size 8 \ + --output_path output/phi-2 \ + --limit 10 \ + --wandb_args project=lm-eval-harness-integration \ + --log_samples +``` + +In the stdout, you will find the link to the W&B run page as well as link to the generated report. You can find an example of this workflow in [examples/visualize-wandb.ipynb](examples/visualize-wandb.ipynb), and an example of how to integrate it beyond the CLI. + +## How to Contribute or Learn More? + +For more information on the library and how everything fits together, check out all of our [documentation pages](https://github.com/EleutherAI/lm-evaluation-harness/tree/main/docs)! We plan to post a larger roadmap of desired + planned library improvements soon, with more information on how contributors can help. + +### Implementing new tasks + +To implement a new task in the eval harness, see [this guide](./docs/new_task_guide.md). + +In general, we follow this priority list for addressing concerns about prompting and other eval details: + +1. If there is widespread agreement among people who train LLMs, use the agreed upon procedure. +2. If there is a clear and unambiguous official implementation, use that procedure. +3. If there is widespread agreement among people who evaluate LLMs, use the agreed upon procedure. +4. If there are multiple common implementations but not universal or widespread agreement, use our preferred option among the common implementations. As before, prioritize choosing from among the implementations found in LLM training papers. + +These are guidelines and not rules, and can be overruled in special circumstances. + +We try to prioritize agreement with the procedures used by other groups to decrease the harm when people inevitably compare runs across different papers despite our discouragement of the practice. Historically, we also prioritized the implementation from [Language Models are Few Shot Learners](https://arxiv.org/abs/2005.14165) as our original goal was specifically to compare results with that paper. + +### Support + +The best way to get support is to open an issue on this repo or join the [EleutherAI Discord server](https://discord.gg/eleutherai). The `#lm-thunderdome` channel is dedicated to developing this project and the `#release-discussion` channel is for receiving support for our releases. If you've used the library and have had a positive (or negative) experience, we'd love to hear from you! + +## Optional Extras + +Extras dependencies can be installed via `pip install -e ".[NAME]"` + +| Name | Use | +| -------------------- | ----------------------------------------------------- | +| api | For using api models (Anthropic, OpenAI API) | +| audiolm_qwen | For running Qwen2 audio models | +| deepsparse | For running NM's DeepSparse models | +| dev | For linting PRs and contributions | +| gptq | For loading models with AutoGPTQ | +| gptqmodel | For loading models with GPTQModel | +| hf_transfer | For speeding up HF Hub file downloads | +| ibm_watsonx_ai | For using IBM watsonx.ai model apis | +| ifeval | For running the IFEval task | +| ipex | For running on optimum-intel ipex backend | +| japanese_leaderboard | For running Japanese LLM Leaderboard tasks | +| longbench | For running LongBench tasks | +| mamba | For loading Mamba SSM models | +| math | For running math task answer checking | +| multilingual | For multilingual tokenizers | +| neuronx | For running on AWS inf2 instances | +| optimum | For running Intel OpenVINO models | +| promptsource | For using PromptSource prompts | +| ruler | For running RULER tasks | +| sae_lens | For using SAELens to steer models | +| sentencepiece | For using the sentencepiece tokenizer | +| sparseml | For using NM's SparseML models | +| sparsify | For using Sparsify to steer models | +| testing | For running library test suite | +| vllm | For loading models with vLLM | +| wandb | For integration with `Weights and Biases` platform | +| zeno | For visualizing results with Zeno | +| -------------------- | ----------------------------------------------------- | +| all | Loads all extras (not recommended) | + +## Cite as + +```text +@misc{eval-harness, + author = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy}, + title = {The Language Model Evaluation Harness}, + month = 07, + year = 2024, + publisher = {Zenodo}, + version = {v0.4.3}, + doi = {10.5281/zenodo.12608602}, + url = {https://zenodo.org/records/12608602} +} +``` diff --git a/lm-evaluation-harness/compute_score.py b/lm-evaluation-harness/compute_score.py new file mode 100644 index 0000000000000000000000000000000000000000..e30e9d3ffcd86c25758b3d2a50d71a5b43c7ac6d --- /dev/null +++ b/lm-evaluation-harness/compute_score.py @@ -0,0 +1,155 @@ +import json +from typing import Dict, Any, Optional, List, Union, Tuple + +import re + +def extract_number_before_year(file_path: str) -> str | None: + """ + ไปŽๆ–‡ไปถ่ทฏๅพ„ไธญๆๅ–ๅนดไปฝ๏ผˆๅ››ไฝๆ•ฐๅญ—๏ผ‰ๅ‰็š„ๆ•ฐๅญ— + + Args: + file_path: ๅพ…ๅค„็†็š„ๆ–‡ไปถ่ทฏๅพ„ๅญ—็ฌฆไธฒ + + Returns: + ๆๅ–ๅˆฐ็š„ๆ•ฐๅญ—ๅญ—็ฌฆไธฒ๏ผˆๅฆ‚"0"๏ผ‰๏ผ›่‹ฅๆœชๆ‰พๅˆฐๅŒน้…้กน๏ผŒ่ฟ”ๅ›žNone + + Example: + >>> path = "/xxx/self_attn_Llama-2-7b-hf_0_2025-11-27.json" + >>> extract_number_before_year(path) + '0' + """ + # ๆญฃๅˆ™ๆจกๅผ่งฃ้‡Š๏ผš + # _(\d+)_ ๅŒน้…ไธ‹ๅˆ’็บฟ+ไธ€ไธฒๆ•ฐๅญ—+ไธ‹ๅˆ’็บฟ๏ผˆๆ‹ฌๅทๆ•่Žทๆ•ฐๅญ—๏ผ‰ + # \d{4} ๅŒน้…ๅ››ไฝๆ•ฐๅญ—๏ผˆๅนดไปฝ๏ผŒๅฆ‚2025ใ€2024๏ผ‰ + pattern = r'_(\d+)_\d{4}' + + # ๆ‰ง่กŒๆญฃๅˆ™ๅŒน้… + match = re.search(pattern, file_path) + + # ่ฟ”ๅ›ž็ป“ๆžœ๏ผšๆœ‰ๅŒน้…ๅˆ™่ฟ”ๅ›žๆ•่Žท็š„ๆ•ฐๅญ—๏ผŒๅฆๅˆ™่ฟ”ๅ›žNone + return match.group(1) if match else None + + + +def read_json_acc_scores( + file_path: str, + target_keys: Optional[Union[str, List[str]]] = None +) -> Union[float, Tuple[Dict[str, float], float]]: + """ + ่ฏปๅ–JSONๆ–‡ไปถไธญ็š„resultsๆ•ฐๆฎ๏ผŒๆๅ–ๅนถ่ฎก็ฎ—acc,noneๅˆ†ๆ•ฐ + + Args: + file_path: JSONๆ–‡ไปถ็š„่ทฏๅพ„ + target_keys: ่ฆๆๅ–็š„key๏ผˆๅ•ไธชๅญ—็ฌฆไธฒๆˆ–ๅญ—็ฌฆไธฒๅˆ—่กจ๏ผ‰๏ผŒไธบNoneๆ—ถๅค„็†ๅ…จ้ƒจresults + + Returns: + - ๅ•ไธชkeyๆ—ถ๏ผš่ฟ”ๅ›žๅฏนๅบ”็š„acc,noneๅˆ†ๆ•ฐ๏ผˆfloat๏ผ‰ + - ๅคšไธชkey/ๅ…จ้ƒจๆ—ถ๏ผš่ฟ”ๅ›ž(ๅ„key็š„accๅˆ†ๆ•ฐๅญ—ๅ…ธ, ๅนณๅ‡ๅˆ†) + + Raises: + FileNotFoundError: ๆ–‡ไปถไธๅญ˜ๅœจๆ—ถๆŠ›ๅ‡บ + json.JSONDecodeError: JSONๆ ผๅผ้”™่ฏฏๆ—ถๆŠ›ๅ‡บ + KeyError: ๆŒ‡ๅฎš็š„keyไธๅญ˜ๅœจๆˆ–็ผบๅฐ‘acc,noneๅญ—ๆฎตๆ—ถๆŠ›ๅ‡บ + """ + + benchmark_metrics = { + "piqa": "acc,none", + "hellaswag": "acc_norm,none", + "arc_challenge": "acc_norm,none", + "boolq": "acc,none", + "winogrande": "acc,none", + } + try: + # ๆ‰“ๅผ€ๅนถ่ฏปๅ–JSONๆ–‡ไปถ + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # ๆๅ–results้ƒจๅˆ† + results = data.get('results', {}) + if not results: + raise KeyError("JSONๆ–‡ไปถไธญๆœชๆ‰พๅˆฐ'results'ๅญ—ๆฎต") + + # ็กฎๅฎš่ฆๅค„็†็š„keys + if target_keys is None: + process_keys = list(results.keys()) # ๅ…จ้ƒจkeys + elif isinstance(target_keys, str): + process_keys = [target_keys] # ๅ•ไธชkey่ฝฌไธบๅˆ—่กจๅค„็† + elif isinstance(target_keys, list): + process_keys = target_keys # ๅˆ—่กจkeys + else: + raise TypeError("target_keysๅ‚ๆ•ฐๅฟ…้กปๆ˜ฏๅญ—็ฌฆไธฒใ€ๅˆ—่กจๆˆ–None") + + # ้ชŒ่ฏkeysๆ˜ฏๅฆๅญ˜ๅœจๅนถๆๅ–acc,noneๅˆ†ๆ•ฐ + acc_scores = {} + conc_ans = 0 + for key in benchmark_metrics: + if key not in results: + continue + + acc_scores[key] = results[key][benchmark_metrics[key]] + conc_ans += results[key][benchmark_metrics[key]] + + # ๅคšไธชkeyๆˆ–ๅ…จ้ƒจ๏ผŒ่ฟ”ๅ›žๅˆ†ๆ•ฐๅญ—ๅ…ธๅ’Œๅนณๅ‡ๅˆ† + # average_score = sum(acc_scores.values()) / len(acc_scores) + return acc_scores, conc_ans + + except FileNotFoundError: + raise FileNotFoundError(f"ๆ–‡ไปถไธๅญ˜ๅœจ: {file_path}") + except json.JSONDecodeError: + raise json.JSONDecodeError("JSONๆ–‡ไปถๆ ผๅผ้”™่ฏฏ", file_path, 0) + +import os +def get_all_paths(folder): + """็”จosๆจกๅ—่Žทๅ–่ทฏๅพ„ไธ‹ๆ‰€ๆœ‰ๆ–‡ไปถ/็›ฎๅฝ•็š„ๅฎŒๆ•ด่ทฏๅพ„""" + all_paths = [] + # ๅ…ˆ่Žทๅ–ๅฝ“ๅ‰็›ฎๅฝ•ไธ‹็š„ๆ‰€ๆœ‰ๅ†…ๅฎน + for name in os.listdir(folder): + full_path = os.path.join(folder, name) + all_paths.append(full_path) + # ๅฆ‚ๆžœๆ˜ฏ็›ฎๅฝ•๏ผŒ้€’ๅฝ’่ฟ›ๅŽป็ปง็ปญ่Žทๅ– + return all_paths + +dir_path = \ + "/mnt/bn/life-mllm/users/cxr/quantization/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer" + # "/mnt/bn/life-mllm/users/cxr/quantization/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp" + # "/mnt/bn/life-mllm/users/cxr/quantization/lm-evaluation-harness/results2/Llama-3.1-8B-quantization-layer" + # "/mnt/bn/life-mllm/users/cxr/quantization/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer" + # "/mnt/bn/life-mllm/users/cxr/quantization/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp" + # "/mnt/bn/life-mllm/users/cxr/quantization/lm-evaluation-harness/results2/Llama-3.1-8B-quantization-layer" + # "/mnt/bn/life-mllm/users/cxr/quantization/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer" + # "/mnt/bn/life-mllm/users/cxr/quantization/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp" + # "/mnt/bn/life-mllm/users/cxr/quantization/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer" + +mode = "self_attn" + + +paths = get_all_paths(dir_path) + +filter_paths = [path for path in paths if mode in path] +print(filter_paths) +total_score = [0 for _ in range(len(filter_paths)-1)] +for filter_path in filter_paths: + final_paths = get_all_paths(filter_path) + idx = filter_path.split("/")[-1] + ans = [] + for final_path in final_paths: + acc_scores, conc_ans = read_json_acc_scores(final_path) + # print(acc_scores) + if conc_ans !=0: + ans.append(conc_ans) + print(idx, sum(ans)/len(ans)) + if int(idx.split("_")[-1]) != -1: + total_score[int(idx.split("_")[-1])] = sum(ans)/len(ans) +print("total_score:", total_score) + +index_value_pairs = list(enumerate(total_score)) +# ๆญฅ้ชค2๏ผšๆŒ‰ๆ•ฐๅ€ผไปŽๅคงๅˆฐๅฐๆŽ’ๅบ๏ผˆkey=lambda x: x[1] ๅ–ๅ…ƒ็ป„็š„็ฌฌไบŒไธชๅ€ผไฝœไธบๆŽ’ๅบไพๆฎ๏ผŒreverse=True ้™ๅบ๏ผ‰ +sorted_pairs = sorted(index_value_pairs, key=lambda x: x[1], reverse=True) + +# ๆญฅ้ชค3๏ผšๆๅ–ๆŽ’ๅบๅŽ็š„็ดขๅผ• +sorted_indices = [pair[0] for pair in sorted_pairs] +print(sorted_indices) +print(' '.join(str(num) for num in sorted_indices[:14])) +# acc_scores, average_score = read_json_acc_scores(path, "piqa") +# print(average_score) + diff --git a/lm-evaluation-harness/docs/API_guide.md b/lm-evaluation-harness/docs/API_guide.md new file mode 100644 index 0000000000000000000000000000000000000000..88e5edc61628b32de65ce82b23e326d49ad7055c --- /dev/null +++ b/lm-evaluation-harness/docs/API_guide.md @@ -0,0 +1,203 @@ +# TemplateAPI Usage Guide + +The `TemplateAPI` class is a versatile superclass designed to facilitate the integration of various API-based language models into the lm-evaluation-harness framework. This guide will explain how to use and extend the `TemplateAPI` class to implement your own API models. If your API implements the OpenAI API you can use the `local-completions` or the `local-chat-completions` (defined [here](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/openai_completions.py)) model types, which can also serve as examples of how to effectively subclass this template. + +## Overview + +The `TemplateAPI` class provides a template for creating API-based model implementations. It handles common functionalities such as: + +- Tokenization (optional) +- Batch processing +- Caching +- Retrying failed requests +- Parsing API responses + +To use this class, you typically need to subclass it and implement specific methods for your API. + +## Key Methods to Implement + +When subclassing `TemplateAPI`, you need to implement the following methods: + +1. `_create_payload`: Creates the JSON payload for API requests. +2. `parse_logprobs`: Parses log probabilities from API responses. +3. `parse_generations`: Parses generated text from API responses. +4. `headers`: Returns the headers for the API request. + +You may also need to override other methods or properties depending on your API's specific requirements. + +> [!NOTE] +> Currently loglikelihood and MCQ based tasks (such as MMLU) are only supported for completion endpoints. Not for chat-completion โ€” those that expect a list of dicts โ€” endpoints! Completion APIs which support instruct tuned models can be evaluated with the `--apply_chat_template` option in order to simultaneously evaluate models using a chat template format while still being able to access the model logits needed for loglikelihood-based tasks. + +## TemplateAPI Arguments + +When initializing a `TemplateAPI` instance or a subclass, you can provide several arguments to customize its behavior. Here's a detailed explanation of some important arguments: + +- `model` or `pretrained` (str): + - The name or identifier of the model to use. + - `model` takes precedence over `pretrained` when both are provided. + +- `base_url` (str): + - The base URL for the API endpoint. + +- `tokenizer` (str, optional): + - The name or path of the tokenizer to use. + - If not provided, it defaults to using the same tokenizer name as the model. + +- `num_concurrent` (int): + - Number of concurrent requests to make to the API. + - Useful for APIs that support parallel processing. + - Default is 1 (sequential processing). + +- `timeout` (int, optional): + - Timeout for API requests in seconds. + - Default is 30. + +- `tokenized_requests` (bool): + - Determines whether the input is pre-tokenized. Defaults to `True`. + - Requests can be sent in either tokenized form (`list[list[int]]`) or as text (`list[str]`, or `str` for batch_size=1). + - For loglikelihood-based tasks, prompts require tokenization to calculate the context length. If `False` prompts are decoded back to text before being sent to the API. + - Not as important for `generate_until` tasks. + - Ignored for chat formatted inputs (list[dict...]) or if tokenizer_backend is None. + +- `tokenizer_backend` (str, optional): + - Required for loglikelihood-based or MCQ tasks. + - Specifies the tokenizer library to use. Options are "tiktoken", "huggingface", or None. + - Default is "huggingface". + +- `max_length` (int, optional): + - Maximum length of input + output. + - Default is 2048. + +- `max_retries` (int, optional): + - Maximum number of retries for failed API requests. + - Default is 3. + +- `max_gen_toks` (int, optional): + - Maximum number of tokens to generate in completion tasks. + - Default is 256 or set in task yaml. + +- `batch_size` (int or str, optional): + - Number of requests to batch together (if the API supports batching). + - Can be an integer or "auto" (which defaults to 1 for API models). + - Default is 1. + +- `seed` (int, optional): + - Random seed for reproducibility. + - Default is 1234. + +- `add_bos_token` (bool, optional): + - Whether to add the beginning-of-sequence token to inputs (when tokenizing). + - Default is False. + +- `custom_prefix_token_id` (int, optional): + - Custom token ID to use as a prefix for inputs. + - If not provided, uses the model's default BOS or EOS token (if `add_bos_token` is True). + +- `verify_certificate` (bool, optional): + - Whether to validate the certificate of the API endpoint (if HTTPS). + - Default is True. + +Example usage: + +```python +class MyAPIModel(TemplateAPI): + def __init__(self, **kwargs): + super().__init__( + model="my-model", + base_url="https://api.mymodel.com/v1/completions", + tokenizer_backend="huggingface", + num_concurrent=5, + max_retries=5, + batch_size=10, + **kwargs + ) + + # Implement other required methods... +``` + +When subclassing `TemplateAPI`, you can override these arguments in your `__init__` method to set default values specific to your API. You can also add additional (potentially user-specified) arguments as needed for your specific implementation. + +## Example Implementation: OpenAI API + +The `OpenAICompletionsAPI` and `OpenAIChatCompletion` ([here](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/openai_completions.py) classes demonstrate how to implement API models using the `TemplateAPI` class. Here's a breakdown of the key components: + +### 1. Subclassing and Initialization + +```python +@register_model("openai-completions") +class OpenAICompletionsAPI(LocalCompletionsAPI): + def __init__( + self, + base_url="https://api.openai.com/v1/completions", + tokenizer_backend="tiktoken", + **kwargs, + ): + super().__init__( + base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs + ) +``` + +### 2. Implementing API Key Retrieval + +```python +@cached_property +def api_key(self): + key = os.environ.get("OPENAI_API_KEY", None) + if key is None: + raise ValueError( + "API key not found. Please set the OPENAI_API_KEY environment variable." + ) + return key +``` + +### 3. Creating the Payload + +```python +def _create_payload( + self, + messages: Union[List[List[int]], List[dict], List[str], str], + generate=False, + gen_kwargs: Optional[dict] = None, + **kwargs, +) -> dict: + if generate: + # ... (implementation for generation) + else: + # ... (implementation for log likelihood) +``` + +### 4. Parsing API Responses + +```python +@staticmethod +def parse_logprobs( + outputs: Union[Dict, List[Dict]], + tokens: List[List[int]] = None, + ctxlens: List[int] = None, + **kwargs, +) -> List[Tuple[float, bool]]: + # ... (implementation) + +@staticmethod +def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]: + # ... (implementation) +``` + +The requests are initiated in the `model_call` or the `amodel_call` methods. + +## Implementing Your Own API Model + +To implement your own API model: + +1. Subclass `TemplateAPI` or one of its subclasses (e.g., `LocalCompletionsAPI`). +2. Override the `__init__` method if you need to set specific parameters. +3. Implement the `_create_payload` and `header` methods to create the appropriate payload for your API. +4. Implement the `parse_logprobs` and `parse_generations` methods to parse your API's responses. +5. Override the `api_key` property if your API requires authentication. +6. Override any other methods as necessary to match your API's behavior. + +## Best Practices + +1. Use the `@register_model` decorator to register your model with the framework (and import it in `lm_eval/models/__init__.py`!). +2. Use environment variables for sensitive information like API keys. +3. Properly handle batching and concurrent requests if supported by your API. diff --git a/lm-evaluation-harness/docs/CONTRIBUTING.md b/lm-evaluation-harness/docs/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..edc58c62670df0548d36472c83c3ef2d251b20d2 --- /dev/null +++ b/lm-evaluation-harness/docs/CONTRIBUTING.md @@ -0,0 +1,83 @@ +# Contributing to LM Evaluation Harness + +Welcome and thank you for your interest in the LM Evaluation Harness! We welcome contributions and feedback and appreciate your time spent with our library, and hope you find it useful! + +## Important Resources + +There are several places information about LM Evaluation Harness is located: + +- Our [documentation pages](https://github.com/EleutherAI/lm-evaluation-harness/tree/main/docs) +- We occasionally use [GitHub Milestones](https://github.com/EleutherAI/lm-evaluation-harness/milestones) to track progress toward specific near-term version releases. +- We maintain a [Project Board](https://github.com/orgs/EleutherAI/projects/25) for tracking current work items and PRs, and for future roadmap items or feature requests. +- Further discussion and support conversations are located in the #lm-thunderdome channel of the [EleutherAI discord](https://discord.gg/eleutherai). + +## Code Style + +LM Evaluation Harness uses [ruff](https://github.com/astral-sh/ruff) for linting via [pre-commit](https://pre-commit.com/). + +You can install linters and dev tools via + +```pip install lm_eval[dev]``` or ```pip install -e ".[dev]"``` + +Then, run + +```pre-commit install``` + +in order to ensure linters and other checks will be run upon committing. + +## Testing + +We use [pytest](https://docs.pytest.org/en/latest/) for running unit tests. All library unit tests can be run via: + +```bash +python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_neuralmagic.py --ignore=tests/models/test_openvino.py +``` + +## Contributor License Agreement + +We ask that new contributors agree to a Contributor License Agreement affirming that EleutherAI has the rights to use your contribution to our library. +First-time pull requests will have a reply added by @CLAassistant containing instructions for how to confirm this, and we require it before merging your PR. + +## Contribution Best Practices + +We recommend a few best practices to make your contributions or reported errors easier to assist with. + +**For Pull Requests:** + +- PRs should be titled descriptively, and be opened with a brief description of the scope and intent of the new contribution. +- New features should have appropriate documentation added alongside them. +- Aim for code maintainability, and minimize code copying. +- If opening a task, try to share test results on the task using a publicly-available model, and if any public results are available on the task, compare to them. + +**For Feature Requests:** + +- Provide a short paragraph's worth of description. What is the feature you are requesting? What is its motivation, and an example use case of it? How does this differ from what is currently supported? + +**For Bug Reports**: + +- Provide a short description of the bug. +- Provide a *reproducible example*--what is the command you run with our library that results in this error? Have you tried any other steps to resolve it? +- Provide a *full error traceback* of the error that occurs, if applicable. A one-line error message or small screenshot snippet is unhelpful without the surrounding context. +- Note what version of the codebase you are using, and any specifics of your environment and setup that may be relevant. + +**For Requesting New Tasks**: + +- Provide a 1-2 sentence description of what the task is and what it evaluates. +- Provide a link to the paper introducing the task. +- Provide a link to where the dataset can be found. +- Provide a link to a paper containing results on an open-source model on the task, for use in comparisons and implementation validation. +- If applicable, link to any codebase that has implemented the task (especially the original publication's codebase, if existent). + +## How Can I Get Involved? + +To quickly get started, we maintain a list of good first issues, which can be found [on our project board](https://github.com/orgs/EleutherAI/projects/25/views/8) or by [filtering GH Issues](https://github.com/EleutherAI/lm-evaluation-harness/issues?q=is%3Aopen+label%3A%22good+first+issue%22+label%3A%22help+wanted%22). These are typically smaller code changes or self-contained features which can be added without extensive familiarity with library internals, and we recommend new contributors consider taking a stab at one of these first if they are feeling uncertain where to begin. + +There are a number of distinct ways to contribute to LM Evaluation Harness, and all are extremely helpful! A sampling of ways to contribute include: + +- **Implementing and verifying new evaluation tasks**: Is there a task you'd like to see LM Evaluation Harness support? Consider opening an issue requesting it, or helping add it! Verifying and cross-checking task implementations with their original versions is also a very valuable form of assistance in ensuring standardized evaluation. +- **Improving documentation** - Improvements to the documentation, or noting pain points / gaps in documentation, are helpful in order for us to improve the user experience of the library and clarity + coverage of documentation. +- **Testing and devops** - We are very grateful for any assistance in adding tests for the library that can be run for new PRs, and other devops workflows. +- **Adding new modeling / inference library integrations** - We hope to support a broad range of commonly-used inference libraries popular among the community, and welcome PRs for new integrations, so long as they are documented properly and maintainable. +- **Proposing or Contributing New Features** - We want LM Evaluation Harness to support a broad range of evaluation usecases. If you have a feature that is not currently supported but desired, feel free to open an issue describing the feature and, if applicable, how you intend to implement it. We would be happy to give feedback on the cleanest way to implement new functionalities and are happy to coordinate with interested contributors via GH discussions or via discord. + +We hope that this has been helpful, and appreciate your interest in contributing! Further questions can be directed to [our Discord](discord.gg/eleutherai). diff --git a/lm-evaluation-harness/docs/README.md b/lm-evaluation-harness/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f040eabaef41b55e9bd8297f3f4653fbc16bf0bc --- /dev/null +++ b/lm-evaluation-harness/docs/README.md @@ -0,0 +1,11 @@ +# Eval Harness Documentation + +Welcome to the docs for the LM Evaluation Harness! + +## Table of Contents + +* To learn about the public interface of the library, as well as how to evaluate via the command line or as integrated into an external library, see the [Interface](./interface.md). +* To learn how to add a new library, API, or model type to the library, as well as a quick explainer on the types of ways to evaluate an LM, see the [Model Guide](./model_guide.md). + * For an extended description of how to extend the library to new model classes served over an API, see the [API Guide](./API_guide.md). +* For a crash course on adding new tasks to the library, see our [New Task Guide](./new_task_guide.md). +* To learn more about pushing the limits of task configuration that the Eval Harness supports, see the [Task Configuration Guide](./task_guide.md). diff --git a/lm-evaluation-harness/docs/chat-template-readme.md b/lm-evaluation-harness/docs/chat-template-readme.md new file mode 100644 index 0000000000000000000000000000000000000000..ae41644e8ba87148b5ffdcb6d81aa40d4b2fde13 --- /dev/null +++ b/lm-evaluation-harness/docs/chat-template-readme.md @@ -0,0 +1,31 @@ +# Chat Template Delimiter Handling Update + +## Overview + +This change modifies how delimiters are handled when applying chat templates in the request construction process for likelihood and multiple-choice based tasks. When `apply_chat_template` is set to `True`, the target delimiter is now set to an empty string instead of using the configured delimiter. + +## Background + +By default, the system uses a target delimiter (typically a whitespace " ") between the context and target text when constructing prompts. The full string is constructed as: + +```text +doc_to_text(doc) + target_delimiter + doc_to_target(doc) +``` + +While this worked well for base models where we wanted the model to predict a single whitespace followed by the answer, chat models have their own formatting conventions that handle spacing differently. + +## The Change + +- When `apply_chat_template=True`, the target delimiter is now empty ("") instead of the default whitespace +- This prevents interference between chat template formatting and the default delimiter system +- Particularly important for multiple choice tasks where the template itself handles spacing + +## Example + +```text +# Before (with default delimiter " ") +Question: What color is the sky?\nAnswer: blue + +# After +Question: What color is the sky?\nAnswer:blue +``` diff --git a/lm-evaluation-harness/docs/decontamination.md b/lm-evaluation-harness/docs/decontamination.md new file mode 100644 index 0000000000000000000000000000000000000000..26b221778244a0f6ad181858d036a1668abab981 --- /dev/null +++ b/lm-evaluation-harness/docs/decontamination.md @@ -0,0 +1,76 @@ +# Decontamination + +## Usage + +The provided directory should contain +the ngram files and info.json produced in "Pile Ngram Generation" further down. + +```bash +python -m lm_eval \ + --model gpt2 \ + --device 0 \ + --tasks sciq +``` + +## Background + +Downstream evaluations test model generalization, and are less useful when test set data also exists in the training set, referred to as leakage or contamination. + +Filtering your training set against the test set is a good first step, however this isn't always possible, as in the case of a new benchmark or one that wasn't considered prior to model training. When training set filtering isn't possible, it is useful to measure the impact of test set leakage by detecting the contaminated test examples and producing a clean version of the benchmark. + +The basis for our decontamination procedure can be found in Appendix C of "Language Models are Few-Shot Learners". OpenAI defined a test document as contaminated if any N-gram overlap existed with any training document. They used a range of N values between 8 and 13 depending on dataset, while we just used 13 for simplicity. + +## Implementation + +Contamination detection can be found in `lm_eval/decontaminate.py` with supporting code in `lm_eval/decontamination/`. + +decontaminate.py does the following: + +1. Build dictionaries of all ngrams and their corresponding evaluation/document ids. +2. Scan through sorted files containing training set n-grams. +3. If a match is found, the corresponding evaluation/document combinations are marked as contaminated. + +`lm_eval/evaluator.py` can then produce a clean version of the benchmark by excluding the results of contaminated documents. For each metric, a clean version will be shown in the results with a "decontaminate" suffix. + +This is disabled by default for new tasks, to support decontamination on a task override the "should_decontaminate" and "doc_to_decontamination_query" methods. For more details see the [task guide](task_guide.md). + +## Pile Ngram Generation + +The relevant scripts can be found in `scripts/clean_training_data`, which also import from +`lm_eval/decontamination/` + +1. git clone https://github.com/EleutherAI/lm-evaluation-harness.git +2. pip install -r requirements.txt +3. Download The Pile from [The Eye](https://the-eye.eu/public/AI/pile/train/) +4. Place pile files in "pile" directory under "lm-evaluation-harness" (or create a symlink) +5. Run generate_13_grams. + +```bash +export PYTHONHASHSEED=0 +python -m scripts/clean_training_data/generate_13_grams \ + -dir path/to/working/directory \ + -n 13 \ + -buckets 500 +``` + +Took approximately 4 days for us. We had the time to wait, but this could be scaled out by doing partial pile scans on multiple instances of this script and merging the relevant buckets. We fixed PYTHONHASHSEED to ensure reproducibility of bucket hashing in case you need to stop and start. + +6. Sort the generated 13-grams. + +```bash +python -m scripts/clean_training_data/sort_13_gram_buckets \ + -dir path/to/working/directory/output +``` + +Took approximately 5 days for us. You could speed this up by spreading the files around to different machines and running the sort script before gathering them together. + +7. Compress the sorted 13 grams files and place them together with info.json. + +This step only takes a few hours. + +```bash +python -m scripts/clean_training_data/compress_and_package \ + -dir path/to/working/directory \ + -output path/to/final/directory \ + -procs 8 +``` diff --git a/lm-evaluation-harness/docs/footguns.md b/lm-evaluation-harness/docs/footguns.md new file mode 100644 index 0000000000000000000000000000000000000000..3343c764e79d8d8023594a74b67366cdd825b450 --- /dev/null +++ b/lm-evaluation-harness/docs/footguns.md @@ -0,0 +1,58 @@ +# Common Pitfalls and Troubleshooting Guide + +This document highlights common pitfalls and troubleshooting tips when using this library. We'll continue to add more tips as we discover them. + +## YAML Configuration Issues + +### Newline Characters in YAML (`\n`) + +**Problem:** When specifying newline characters in YAML, they may be interpreted incorrectly depending on how you format them. + +```yaml +# โŒ WRONG: Single quotes don't process escape sequences +generation_kwargs: + until: ['\n'] # Gets parsed as the literal characters '\' and 'n' i.e "\\n" + +``` +```yaml +# โœ… RIGHT: Use double quotes for escape sequences +generation_kwargs: + until: ["\n"] # Gets parsed as an actual newline character + +``` + +**Solutions:** +- Use double quotes for strings containing escape sequences +- For multiline content, use YAML's block scalars (`|` or `>`) +- When generating YAML programmatically, be careful with how template engines handle escape sequences + +### Quoting in YAML + +**When to use different types of quotes:** + +- **No quotes**: Simple values (numbers, booleans, alphanumeric strings without special characters) + ```yaml + simple_value: plain text + number: 42 + + ``` + +- **Single quotes (')**: + - Preserves literal values + - Use when you need special characters to be treated literally + - Escape single quotes by doubling them: `'It''s working'` + ```yaml + literal_string: 'The newline character \n is not processed here' + path: 'C:\Users\name' # Backslashes preserved + + ``` + +- **Double quotes (")**: + - Processes escape sequences like `\n`, `\t`, etc. + - Use for strings that need special characters interpreted + - Escape double quotes with backslash: `"He said \"Hello\""` + ```yaml + processed_string: "First line\nSecond line" # Creates actual newline + unicode: "Copyright symbol: \u00A9" # Unicode character + + ``` diff --git a/lm-evaluation-harness/docs/interface.md b/lm-evaluation-harness/docs/interface.md new file mode 100644 index 0000000000000000000000000000000000000000..570d96ddeca926fcc6cd9d776d92ae7c57323d92 --- /dev/null +++ b/lm-evaluation-harness/docs/interface.md @@ -0,0 +1,170 @@ +# User Guide + +This document details the interface exposed by `lm-eval` and provides details on what flags are available to users. + +## Command-line Interface + +A majority of users run the library by cloning it from Github, installing the package as editable, and running the `python -m lm_eval` script. + +Equivalently, running the library can be done via the `lm-eval` entrypoint at the command line. + +This mode supports a number of command-line arguments, the details of which can also be seen via running with `-h` or `--help`: + +- `--model` : Selects which model type or provider is evaluated. Must be a string corresponding to the name of the model type/provider being used. See [the main README](https://github.com/EleutherAI/lm-evaluation-harness/tree/main#model-apis-and-inference-servers) for a full list of enabled model names and supported libraries or APIs. + +- `--model_args` : Controls parameters passed to the model constructor. Accepts a string containing comma-separated keyword arguments to the model class of the format `"arg1=val1,arg2=val2,..."`, such as, for example `--model_args pretrained=EleutherAI/pythia-160m,dtype=float32`. For a full list of what keyword arguments, see the initialization of the `lm_eval.api.model.LM` subclass, e.g. [`HFLM`](https://github.com/EleutherAI/lm-evaluation-harness/blob/365fcda9b85bbb6e0572d91976b8daf409164500/lm_eval/models/huggingface.py#L66) + +- `--tasks` : Determines which tasks or task groups are evaluated. Accepts a comma-separated list of task names or task group names. Must be solely comprised of valid tasks/groups. A list of supported tasks can be viewed with `--tasks list`. + +- `--num_fewshot` : Sets the number of few-shot examples to place in context. Must be an integer. + +- `--gen_kwargs` : takes an arg string in same format as `--model_args` and creates a dictionary of keyword arguments. These will be passed to the models for all called `generate_until` (free-form or greedy generation task) tasks, to set options such as the sampling temperature or `top_p` / `top_k`. For a list of what args are supported for each model type, reference the respective library's documentation (for example, the documentation for `transformers.AutoModelForCausalLM.generate()`.) These kwargs will be applied to all `generate_until` tasks called--we do not currently support unique gen_kwargs or batch_size values per task in a single run of the library. To control these on a per-task level, set them in that task's YAML file. + +- `--batch_size` : Sets the batch size used for evaluation. Can be a positive integer or `"auto"` to automatically select the largest batch size that will fit in memory, speeding up evaluation. One can pass `--batch_size auto:N` to re-select the maximum batch size `N` times during evaluation. This can help accelerate evaluation further, since `lm-eval` sorts documents in descending order of context length. + +- `--max_batch_size` : Sets the maximum batch size to try to fit in memory, if `--batch_size auto` is passed. + +- `--device` : Sets which device to place the model onto. Must be a string, for example, `"cuda", "cuda:0", "cpu", "mps"`. Defaults to "cuda", and can be ignored if running multi-GPU or running a non-local model type. + +- `--output_path` : A string of the form `dir/file.jsonl` or `dir/`. Provides a path where high-level results will be saved, either into the file named or into the directory named. If `--log_samples` is passed as well, then per-document outputs and metrics will be saved into the directory as well. + +- `--log_samples` : If this flag is passed, then the model's outputs, and the text fed into the model, will be saved at per-document granularity. Must be used with `--output_path`. + +- `--limit` : Accepts an integer, or a float between 0.0 and 1.0 . If passed, will limit the number of documents to evaluate to the first X documents (if an integer) per task or first X% of documents per task. Useful for debugging, especially on costly API models. + +- `--use_cache` : Should be a path where a sqlite db file can be written to. Takes a string of format `/path/to/sqlite_cache_` in order to create a cache db at `/path/to/sqlite_cache_rank{i}.db` for each process (0-NUM_GPUS). This allows results of prior runs to be cached, so that there is no need to re-run results in order to re-score or re-run a given (model, task) pair again. + +- `--cache_requests` : Can be "true", "refresh", or "delete". "true" means that the cache should be used. "refresh" means that you wish to regenerate the cache, which you should run if you change your dataset configuration for a given task. "delete" will delete the cache. Cached files are stored under lm_eval/cache/.cache unless you specify a different path via the environment variable: `LM_HARNESS_CACHE_PATH`. e.g. `LM_HARNESS_CACHE_PATH=~/Documents/cache_for_lm_harness`. + +- `--check_integrity` : If this flag is used, the library tests for each task selected are run to confirm task integrity. + +- `--write_out` : Used for diagnostic purposes to observe the format of task documents passed to a model. If this flag is used, then prints the prompt and gold target string for the first document of each task. + +- `--show_config` : If used, prints the full `lm_eval.api.task.TaskConfig` contents (non-default settings the task YAML file) for each task which was run, at the completion of an evaluation. Useful for when one is modifying a task's configuration YAML locally to transmit the exact configurations used for debugging or for reproducibility purposes. + +- `--include_path` : Accepts a path to a folder. If passed, then all YAML files containing `lm-eval` compatible task configurations will be added to the task registry as available tasks. Used for when one is writing config files for their own task in a folder other than `lm_eval/tasks/`. + +- `--system_instruction`: Specifies a system instruction string to prepend to the prompt. + +- `--apply_chat_template` : This flag specifies whether to apply a chat template to the prompt. It can be used in the following ways: + - `--apply_chat_template` : When used without an argument, applies the only available chat template to the prompt. For Hugging Face models, if no dedicated chat template exists, the default chat template will be applied. + - `--apply_chat_template template_name` : If the model has multiple chat templates, apply the specified template to the prompt. + + For Hugging Face models, the default chat template can be found in the [`default_chat_template`](https://github.com/huggingface/transformers/blob/fc35907f95459d7a6c5281dfadd680b6f7b620e3/src/transformers/tokenization_utils_base.py#L1912) property of the Transformers Tokenizer. + +- `--fewshot_as_multiturn` : If this flag is on, the Fewshot examples are treated as a multi-turn conversation. Questions are provided as user content and answers are provided as assistant responses. Requires `--num_fewshot` to be set to be greater than 0, and `--apply_chat_template` to be on. + +- `--predict_only`: Generates the model outputs without computing metrics. Use with `--log_samples` to retrieve decoded results. + +- `--seed`: Set seed for python's random, numpy and torch. Accepts a comma-separated list of 3 values for python's random, numpy, and torch seeds, respectively, or a single integer to set the same seed for all three. The values are either an integer or 'None' to not set the seed. Default is `0,1234,1234` (for backward compatibility). E.g. `--seed 0,None,8` sets `random.seed(0)` and `torch.manual_seed(8)`. Here numpy's seed is not set since the second value is `None`. E.g, `--seed 42` sets all three seeds to 42. + +- `--wandb_args`: Tracks logging to Weights and Biases for evaluation runs and includes args passed to `wandb.init`, such as `project` and `job_type`. Full list [here](https://docs.wandb.ai/ref/python/init). e.g., ```--wandb_args project=test-project,name=test-run```. Also allows for the passing of the step to log things at (passed to `wandb.run.log`), e.g., `--wandb_args step=123`. + +- `--hf_hub_log_args` : Logs evaluation results to Hugging Face Hub. Accepts a string with the arguments separated by commas. Available arguments: + - `hub_results_org` - organization name on Hugging Face Hub, e.g., `EleutherAI`. If not provided, the results will be pushed to the owner of the Hugging Face token, + - `hub_repo_name` - repository name on Hugging Face Hub (deprecated, `details_repo_name` and `results_repo_name` should be used instead), e.g., `lm-eval-results`, + - `details_repo_name` - repository name on Hugging Face Hub to store details, e.g., `lm-eval-results`, + - `results_repo_name` - repository name on Hugging Face Hub to store results, e.g., `lm-eval-results`, + - `push_results_to_hub` - whether to push results to Hugging Face Hub, can be `True` or `False`, + - `push_samples_to_hub` - whether to push samples results to Hugging Face Hub, can be `True` or `False`. Requires `--log_samples` to be set, + - `public_repo` - whether the repository is public, can be `True` or `False`, + - `leaderboard_url` - URL to the leaderboard, e.g., `https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard`. + - `point_of_contact` - Point of contact for the results dataset, e.g., `yourname@example.com`. + - `gated` - whether to gate the details dataset, can be `True` or `False`. + +- `--metadata`: JSON string to pass to TaskConfig. Used for some tasks which require additional metadata to be passed for processing. E.g., `--metadata '{"key": "value"}'`. + +## External Library Usage + +We also support using the library's external API for use within model training loops or other scripts. + +`lm_eval` supplies two functions for external import and use: `lm_eval.evaluate()` and `lm_eval.simple_evaluate()`. + +`simple_evaluate()` can be used by simply creating an `lm_eval.api.model.LM` subclass that implements the methods described in the [Model Guide](https://github.com/EleutherAI/lm-evaluation-harness/tree/main/docs/model_guide.md), and wrapping your custom model in that class as follows: + +```python +import lm_eval +from lm_eval.utils import setup_logging +... +# initialize logging +setup_logging("DEBUG") # optional, but recommended; or you can set up logging yourself +my_model = initialize_my_model() # create your model (could be running finetuning with some custom modeling code) +... +# instantiate an LM subclass that takes your initialized model and can run +# - `Your_LM.loglikelihood()` +# - `Your_LM.loglikelihood_rolling()` +# - `Your_LM.generate_until()` +lm_obj = Your_LM(model=my_model, batch_size=16) + +# indexes all tasks from the `lm_eval/tasks` subdirectory. +# Alternatively, you can set `TaskManager(include_path="path/to/my/custom/task/configs")` +# to include a set of tasks in a separate directory. +task_manager = lm_eval.tasks.TaskManager() + +# Setting `task_manager` to the one above is optional and should generally be done +# if you want to include tasks from paths other than ones in `lm_eval/tasks`. +# `simple_evaluate` will instantiate its own task_manager if it is set to None here. +results = lm_eval.simple_evaluate( # call simple_evaluate + model=lm_obj, + tasks=["taskname1", "taskname2"], + num_fewshot=0, + task_manager=task_manager, + ... +) +``` + +See the `simple_evaluate()` and `evaluate()` functions in [lm_eval/evaluator.py](../lm_eval/evaluator.py#:~:text=simple_evaluate) for a full description of all arguments available. All keyword arguments to simple_evaluate share the same role as the command-line flags described previously. + +Additionally, the `evaluate()` function offers the core evaluation functionality provided by the library, but without some of the special handling and simplification + abstraction provided by `simple_evaluate()`. + +As a brief example usage of `evaluate()`: + +```python +import lm_eval + +# suppose you've defined a custom lm_eval.api.Task subclass in your own external codebase +from my_tasks import MyTask1 +... + +# create your model (could be running finetuning with some custom modeling code) +my_model = initialize_my_model() +... + +# instantiate an LM subclass that takes your initialized model and can run +# - `Your_LM.loglikelihood()` +# - `Your_LM.loglikelihood_rolling()` +# - `Your_LM.generate_until()` +lm_obj = Your_LM(model=my_model, batch_size=16) + +# optional: the task_manager indexes tasks including ones +# specified by the user through `include_path`. +task_manager = lm_eval.tasks.TaskManager( + include_path="/path/to/custom/yaml" + ) + +# To get a task dict for `evaluate` +task_dict = lm_eval.tasks.get_task_dict( + [ + "mmlu", # A stock task + "my_custom_task", # A custom task + { + "task": ..., # A dict that configures a task + "doc_to_text": ..., + }, + MyTask1 # A task object from `lm_eval.task.Task` + ], + task_manager # A task manager that allows lm_eval to + # load the task during evaluation. + # If none is provided, `get_task_dict` + # will instantiate one itself, but this + # only includes the stock tasks so users + # will need to set this if including + # custom paths is required. + ) + +results = evaluate( + lm=lm_obj, + task_dict=task_dict, + ... +) +``` diff --git a/lm-evaluation-harness/docs/model_guide.md b/lm-evaluation-harness/docs/model_guide.md new file mode 100644 index 0000000000000000000000000000000000000000..3c122ffec54a4bf1f26614148a3e100eb3a8402c --- /dev/null +++ b/lm-evaluation-harness/docs/model_guide.md @@ -0,0 +1,192 @@ +# New Model Guide + +This guide may be of special interest to users who are using the library outside of the repository, via installing the library via pypi and calling `lm_eval.evaluator.evaluate()` to evaluate an existing model. + +In order to properly evaluate a given LM, we require implementation of a wrapper class subclassing the `lm_eval.api.model.LM` class, that defines how the Evaluation Harness should interface with your model. This guide walks through how to write this `LM` subclass via adding it to the library! + +## Setup + +To get started contributing, go ahead and fork the main repo, clone it, create a branch with the name of your model, and install the project requirements in your environment: + +```sh +# After forking... +git clone https://github.com//lm-evaluation-harness.git +cd lm-evaluation-harness +git checkout -b +pip install -e ".[dev]" +``` + +Now, we'll create a new file where we'll be adding our model: + +```sh +touch lm_eval/models/.py +``` + +**Tip: this filename should not shadow package names! For example, naming your file `anthropic.py` is disallowed since the API's name on pypi is `anthropic`, but naming it `anthropic_llms.py` works with no problems.** + +## Interface + +All models must subclass the `lm_eval.api.model.LM` class. + +The LM class enforces a common interface via which we can extract responses from a model: + +```python +class MyCustomLM(LM): + #... + def loglikelihood(self, requests: list[Instance]) -> list[tuple[float, bool]]: + #... + + + def loglikelihood_rolling(self, requests: list[Instance]) -> list[tuple[float, bool]]: + #... + + + def generate_until(self, requests: list[Instance]) -> list[str]: + #... + #... +``` + +Where `Instance` is a dataclass defined in [`lm_eval.api.instance`](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/api/instance.py) with property `args` of request-dependent type signature described below. + +We support three types of requests, consisting of different interactions / measurements with an autoregressive LM. + +All three request types take as input `requests` of type `list[Instance]` that have a matching `Instance.request_type` to the method name. + +- `generate_until` + - Each request contains `Instance.args : Tuple[str, dict]` containing 1. an input string to the LM and 2. a dictionary of keyword arguments used to control generation parameters. + - Using this input and these generation parameters, text will be sampled from the language model (typically until a maximum output length or specific stopping string sequences--for example, `{"until": ["\n\n", "."], "max_gen_toks": 128}`). + - The generated output text from the model will then be returned. + +- `loglikelihood` + - Each request contains `Instance.args : Tuple[str, str]` containing 1. an input string to the LM and 2. a target string on which the loglikelihood of the LM producing this target, conditioned on the input, will be returned. + - Each request will have, as result, `(ll, is_greedy): Tuple[float, int]` returned, where `ll` is a floating point number representing the log probability of generating the target string conditioned on the input, and `is_greedy` being either the value `0` or `1`, with it being `1` if and only if the target string *would be generated by greedy sampling from the LM* (that is, if the target string is the *most likely* N-token string to be output by the LM given the input. ) + +- `loglikelihood_rolling` + - Each request contains `Instance.args : Tuple[str]`, which is an input string to the model whose *entire* loglikelihood, conditioned on purely the EOT token, will be calculated. + - This is used to evaluate *perplexity* on a data distribution. + - It should return `(ll,) : Tuple[float]` , a.k.a. solely the *loglikelihood* of producing each piece of text given no starting input. + +To allow a model to be evaluated on all types of tasks, you will need to implement these three types of measurements (note that `loglikelihood_rolling` is a special case of `loglikelihood`). For a reference implementation, check out `lm_eval/models/huggingface.py` ! Additionally, check out `lm_eval.api.model.TemplateLM` for a class that abstracts away some commonly used functions across LM subclasses, or see if your model would lend itself well to subclassing the `lm_eval.models.huggingface.HFLM` class and overriding just the initialization or a couple methods! + +**Tip: be careful of indexing in loglikelihood!** + +LMs take in tokens in position `[0 1 2 ... N]` and output a probability distribution for token position `N+1`. We provide a simplified graphic here, excerpted from `huggingface.py`: + +```text +# how this all works (illustrated on a causal decoder-only setup): +# CTX CONT +# inp 0 1 2 3|4 5 6 7 8 9 <- last token is deleted by inp[:, :-1] +# model \ \ +# logits 1 2 3|4 5 6 7 8 9 <- the ctx half gets tossed out by the +# cont_toks 4 5 6 7 8 9 [:, -len(continuation_enc):, :self.vocab_size] slice +``` + +The final token of the target is not passed into the LM, because we want the LM's predictions *up to but not past* that final target token. For more information, check out https://github.com/EleutherAI/lm-evaluation-harness/issues/942 . + +## Registration + +Congrats on implementing your model! Now it's time to test it out. + +To make your model usable via the command line interface to `lm-eval` using `python -m lm_eval`, you'll need to tell `lm-eval` what your model's name is. + +This is done via a *decorator*, `lm_eval.api.registry.register_model`. Using `register_model()`, one can both tell the package what the model's name(s) to be used are when invoking it with `python -m lm_eval --model ` and alert `lm-eval` to the model's existence. + +```python +from lm_eval.api.registry import register_model + +@register_model("", "") +class MyCustomLM(LM): +``` + +Using this decorator results in the class being added to an accounting of the usable LM types maintained internally to the library at `lm_eval.api.registry.MODEL_REGISTRY`. See `lm_eval.api.registry` for more detail on what sorts of registries and decorators exist in the library! + +**Tip: be sure to import your model in `lm_eval/models/__init__.py!`** + +## Testing + +We also recommend that new model contributions be accompanied by short tests of their 3 core functionalities, at minimum. To see an example of such tests, look at https://github.com/EleutherAI/lm-evaluation-harness/blob/35bdecd379c0cefad6897e67db892f4a6026a128/tests/test_ggml.py . + +## Chat Templating + +Many models are fine-tuned with a [Chat Template](https://huggingface.co/docs/transformers/main/en/chat_templating) in order to enable back-and-forth interaction between a "User"'s queries and the model (often called "Assistant")'s responses. It can be desirable to evaluate fine-tuned models on evaluation tasks while wrapped in the conversational format they expect. + +In order to make your model optionally compatible with a chat format, three additional methods must be implemented: + +```python +class MyCustomLM(LM): + #... + @property + def tokenizer_name(self) -> str: + """ + Return the name of the model's tokenizer and/or the accompanying chat template. + The returned string is used to cache requests. + + Returns: + str: The name of the model's tokenizer and/or chat template. + """ + + def chat_template(self, chat_template: Union[bool, str] = False) -> str: + """ + Get the appropriate chat template for the model based on the `chat_template` argument. + + This method returns the chat template string to build the prompt from a chat history. + The chat template is saved in the evaluation results for reproducibility. + Boolean arguments should be used with models that have only one chat template, + while string arguments are used with models that have multiple chat templates. + For the reference implementation, see HFLM class in `lm_eval.models.huggingface`. + + Args: + chat_template (Union[bool, str]): Specifies whether to apply a chat template: + - If False: Do not apply any chat template. + - If True: Apply the default chat template. + - If str: Apply the specified chat template by name. + + Returns: + str: The selected chat template in Jinja format. + """ + + def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str: + """ + Process a chat history to create a string that can be tokenized and input into the model. + + Args: + chat_history (List[Dict[str, str]]): A list of dictionaries representing the chat history, + where each dictionary has "role" and "content" keys. + + Returns: + str: A string representing the chat history that can be tokenized and fed into the model. + """ +``` + +- `apply_chat_template` + - This method performs the bulk of the work required for chat-formatting. + - As input, a `chat_history: List[Dict[str, str]]` is passed in. This is a transcript of a conversation of a form similar to + + ```text + [ + {"system": }, + {"user": } + {"assistant": }, + # ... more few-shot examples, potentially + {"user": }, + ] + ``` + + which can then be converted into a string input. + - The output is a string representing this conversation that can be fed into the model. + - For example, this consists of simply calling `tokenizer.apply_chat_template` for HFLM--see the implementation there for reference. +- `tokenizer_name` + - LM Eval Harness supports [caching requests](https://github.com/EleutherAI/lm-evaluation-harness/blob/4902aaaf1f374682f95ac25fe2e13b23faddc91a/lm_eval/__main__.py#L140) that are sent to a model, for faster setup when repeating an already-performed evaluation. + - However, we don't want to use the cache of chat transcripts rendered using one chat template or system prompt to send to a model with a different template! So, we use this `lm.tokenizer_name` string to distinguish caches for a given model (and chat template) from one another. +- `chat_template` + - Chat templates are typically provided as a Jinja template string or a string formatted with str.format to include user and assistant messages in a single prompt. This template string is saved in the evaluation results to ensure reproducibility. + +If not implemented for a given model type, the flags `--apply_chat_template` , `--fewshot_as_multiturn`, and `--system_instruction` cannot be used. + +## Other + +**Pro tip**: In order to make the Evaluation Harness overestimate total runtimes rather than underestimate it, HuggingFace models come in-built with the ability to provide responses on data points in *descending order by total input length* via `lm_eval.utils.Reorderer`. Take a look at `lm_eval.models.hf_causal.HFLM` to see how this is done, and see if you can implement it in your own model! + +## Conclusion + +After reading this guide, you should be able to add new model APIs or implementations to the Eval Harness library! diff --git a/lm-evaluation-harness/docs/new_task_guide.md b/lm-evaluation-harness/docs/new_task_guide.md new file mode 100644 index 0000000000000000000000000000000000000000..d8601939dc8c4bc9d1d12924fee9bbd4991bed15 --- /dev/null +++ b/lm-evaluation-harness/docs/new_task_guide.md @@ -0,0 +1,521 @@ +# New Task Guide + +`lm-evaluation-harness` is a framework that strives to support a wide range of zero- and few-shot evaluation tasks on autoregressive language models (LMs). + +This documentation page provides a walkthrough to get started creating your own task, in `lm-eval` versions v0.4.0 and later. + +A more interactive tutorial is available as a Jupyter notebook [here](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/examples/lm-eval-overview.ipynb). + +## Setup + +If you haven't already, go ahead and fork the main repo, clone it, create a branch with the name of your task, and install the project requirements in your environment: + +```sh +# After forking... +git clone https://github.com//lm-evaluation-harness.git +cd lm-evaluation-harness +git checkout -b +pip install -e ".[dev]" +``` + +In this document, we'll walk through the basics of implementing a static benchmark evaluation in two formats: a *generative* task which requires sampling text from a model, such as [`gsm8k`](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/gsm8k/gsm8k.yaml), and a *discriminative*, or *multiple choice*, task where the model picks the most likely of several fixed answer choices, such as [`sciq`](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/sciq/sciq.yaml). + +## Creating a YAML file + +To implement a new standard task, we'll need to write a YAML file which configures our task logic. We start by making a new empty YAML file. This file can have any name, but we recommend placing it in a subfolder of `lm_eval/tasks` titled by the dataset or task's shorthand name: for example, + +```sh +touch lm_eval/tasks//.yaml +``` + +Or, copy the template subfolder we provide from `templates/new_yaml_task`: + +```sh +cp -r templates/new_yaml_task lm_eval/tasks/ +``` + +and rename the folders and YAML file(s) as desired. + +### Selecting and configuring a dataset + +All data downloading and management is handled through the HuggingFace (**HF**) [`datasets`](https://github.com/huggingface/datasets) API. So, the first thing you should do is check to see if your task's dataset is already provided in their catalog [here](https://huggingface.co/datasets). If it's not in there, please consider adding it to their Hub to make it accessible to a wider user base by following their [new dataset guide](https://github.com/huggingface/datasets/blob/main/ADD_NEW_DATASET.md) +. +> [!TIP] +> To test your task, we recommend using verbose logging using `export LOGLEVEL = DEBUG` in your shell before running the evaluation script. This will help you debug any issues that may arise. +Once you have a HuggingFace dataset prepared for your task, we want to assign our new YAML to use this dataset: + +```yaml +dataset_path: ... # the name of the dataset on the HF Hub. +dataset_name: ... # the dataset configuration to use. Leave `null` if your dataset does not require a config to be passed. See https://huggingface.co/docs/datasets/load_hub#configurations for more info. +dataset_kwargs: null # any extra keyword arguments that should be passed to the dataset constructor, e.g. `data_dir`. +``` + +Next, we'd like to tell our task what the dataset's train, validation, and test splits are named, if they exist: + +```yaml +training_split: +validation_split: +test_split: +``` + +Tests will run on the `test_split` if it is available, and otherwise evaluate on the `validation_split`. + +We can also specify from which split the task should retrieve few-shot examples via: + +```yaml +fewshot_split: +``` + +or by hardcoding them, either using the following in the yaml file: + +```yaml +fewshot_config: + sampler: first_n + samples: [ + {}, + {}, + ] +``` + +or by adding the function `list_fewshot_samples` in the associated utils.py file: + +```python +def list_fewshot_samples() -> list[dict]: + return [{}, {}] +``` + +See `lm_eval/tasks/minerva_math/minerva_math_algebra.yaml` for an example of the latter, and `lm_eval/tasks/gsm8k/gsm8k-cot.yaml` for an example of the former. + +In this case, each sample must contain the same fields as the samples in the above sets--for example, if `doc_to_text` expects an `input` field when rendering input prompts, these provided samples must include an `input` key. + +If neither above options are not set, we will default to train/validation/test sets, in that order. + +Finally, our dataset may not be already in the exact format we want. Maybe we have to strip whitespace and special characters via a regex from our dataset's "question" field! Or maybe we just want to rename its columns to match a convention we'll be using for our prompts. + +Let's create a python file in the directory where we're writing our YAML file: + +```bash +touch lm_eval/tasks//utils.py +``` + +Now, in `utils.py` we'll write a function to process each split of our dataset (the following example is drawn from [the `hellaswag` task](../lm_eval/tasks/hellaswag/utils.py)): + +```python +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize() + out_doc = { + "query": preprocess(doc["activity_label"] + ": " + ctx), + "choices": [preprocess(ending) for ending in doc["endings"]], + "gold": int(doc["label"]), + } + return out_doc + + return dataset.map(_process_doc) +``` + +Now, in our YAML config file we'll use the `!function` constructor, and tell the config where our imported Python function will come from. At runtime, before doing anything else we will preprocess our dataset according to this function! + +```yaml +process_docs: !function utils.process_docs +``` + +### Using Local Datasets + +To load a local dataset for evaluation, you can specify data files in the `dataset_kwargs` field, such as the following for JSON files: + +```yaml +dataset_path: json +dataset_name: null +dataset_kwargs: + data_files: /path/to/my/json +``` + +Or with files already split into separate directories: + +```yaml +dataset_path: arrow +dataset_kwargs: + data_files: + train: /path/to/arrow/train/data-00000-of-00001.arrow + validation: /path/to/arrow/validation/data-00000-of-00001.arrow +``` + +Alternatively, if you have previously downloaded a dataset from huggingface hub (using `save_to_disk()`) and wish to use the local files, you will need to use `data_dir` under `dataset_kwargs` to point to where the directory is. + +```yaml +dataset_path: hellaswag +dataset_kwargs: + data_dir: hellaswag_local/ +``` + +You can also set `dataset_path` as a directory path in your local system. This will assume that there is a loading script with the same name as the directory. [See datasets docs](https://huggingface.co/docs/datasets/loading#local-loading-script). + +## Writing a Prompt Template + +The next thing we need to do is decide what format to use when presenting the data to the LM. This is our **prompt**, where we'll define both an input and output format. + +To write a prompt, users will use `doc_to_text`, `doc_to_target`, and `doc_to_choice` (Optional when certain conditions are met). + +`doc_to_text` defines the input string a model will be given while `doc_to_target` and `doc_to_choice` will be used to generate the target text. `doc_to_target` can be either a text string that refers to the target string or an integer that refers to the index of the correct label. When it is set as an index, `doc_to_choice` must also be set with the appropriate list of possible choice strings. + +### Basic prompts + +If a dataset is straightforward enough, users can enter the feature name directly. This assumes that no preprocessing is required. For example in [Swag](https://github.com/EleutherAI/lm-evaluation-harness/blob/1710b42d52d0f327cb0eb3cb1bfbbeca992836ca/lm_eval/tasks/swag/swag.yaml#L10-L11), `doc_to_text` and `doc_to_target` given the name of one of the feature each. + +```yaml +doc_to_text: startphrase +doc_to_target: label +``` + +Hard-coding is also possible as is the case in [SciQ](https://github.com/EleutherAI/lm-evaluation-harness/blob/1710b42d52d0f327cb0eb3cb1bfbbeca992836ca/lm_eval/tasks/sciq/sciq.yaml#L11). + +```yaml +doc_to_target: 3 +``` + +`doc_to_choice` can be directly given a list of text as option (See [Toxigen](https://github.com/EleutherAI/lm-evaluation-harness/blob/1710b42d52d0f327cb0eb3cb1bfbbeca992836ca/lm_eval/tasks/toxigen/toxigen.yaml#L11)) + +```yaml +doc_to_choice: ['No', 'Yes'] +``` + +if a dataset feature is already a list, you can set the name of the feature as `doc_to_choice` (See [Hellaswag](https://github.com/EleutherAI/lm-evaluation-harness/blob/e0eda4d3ffa10e5f65e0976161cd134bec61983a/lm_eval/tasks/hellaswag/hellaswag.yaml#L13)) + +```yaml +doc_to_choice: choices +``` + +### Writing a prompt with Jinja 2 + +We support the [Jinja 2](https://jinja.palletsprojects.com/en/3.1.x/) templating language for writing prompts. In practice, this means you can take your dataset's columns and do many basic string manipulations to place each document into prompted format. + +Take for example the dataset `super_glue/boolq`. As input, we'd like to use the features `passage` and `question` and string them together so that for a sample line `doc`, the model sees something in the format of: + +```text +doc["passage"] +Question: doc["question"]? +Answer: +``` + +We do this by [writing](https://github.com/EleutherAI/lm-evaluation-harness/blob/1710b42d52d0f327cb0eb3cb1bfbbeca992836ca/lm_eval/tasks/super_glue/boolq/default.yaml#L9C1-L9C61) + +```yaml +doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:" +``` + +Such that `{{passage}}` will be replaced by `doc["passage"]` and `{{question}}` with `doc["question"]` when rendering the prompt template. + +Our intended output is for the model to predict a single whitespace, and then the answer to the question. We do this via: + +```yaml +doc_to_target: "{{answer}}" +``` + +#### Multiple choice format + +For tasks which are multiple choice (a fixed, finite set of label words per each document) and evaluated via comparing loglikelihoods of all label words (the `multiple_choice` task output type) we enforce a particular convention on prompt format. + +> [!WARNING] +> We add `target_delimiter` between input and target which defaults to " ", such that the full input-output string is `doc_to_text(doc) + target_delimiter + doc_to_target(doc)`. `doc_to_text` and `doc_to_target` should not contain trailing right or left whitespace, respectively. For multiple choice the target will be each choice index concatenated with the delimiter. + +An annotated example in the case of SciQ is as follows: + +```yaml +doc_to_text: "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:" # This is the input portion of the prompt for this doc. It will have " {{choice}}" appended to it as target for each choice in answer_choices. +doc_to_target: 3 # this contains the index into the answer choice list of the correct answer. +doc_to_choice: "{{[distractor1, distractor2, distractor3, correct_answer]}}" +``` + +Task implementers are thus able to decide what the answer choices should be for a document, and what prompt format to use. + +The label index can also be sourced from a feature directly. For example in `superglue/boolq`, the label index if defined in the feature `label`. We can set `doc_to_target` as simply `label`. The options or verbalizers can be written in the form of a list `["no", "yes"]` that will correspond to the label index. + +```yaml +doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:" +doc_to_target: label +doc_to_choice: ["no", "yes"] +``` + +### Using Python Functions for Prompts + +There may be cases where the prompt we want to implement is easier expressed in Python instead of Jinja 2. For this, we can use Python helper functions that are defined in the YAML config. It should be noted that the function script must be in the same directory as the yaml. + +A good example is WikiText that requires a lot of regex rules to clean the samples. + +```python +def wikitext_detokenizer(doc): + string = doc["page"] + # contractions + string = string.replace("s '", "s'") + string = re.sub(r"/' [0-9]/", r"/'[0-9]/", string) + ... + string = string.replace(" 's", "'s") + + return string +``` + +We can load this function in `doc_to_target` by using a `!function` operator after `doc_to_target` and followed by `.`. In the file [wikitext.yaml](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/wikitext/wikitext.yaml) we write: + +```yaml +doc_to_target: !function preprocess_wikitext.wikitext_detokenizer +``` + +### Importing a Prompt from Promptsource + +[Promptsource](https://github.com/bigscience-workshop/promptsource/tree/main/promptsource) is a great repository for crowdsourced prompts for many datasets. We can load these prompts easily by using the `use_prompt` argument and filling it with the format `"promptsource:"`. To use this, `doc_to_text` and `doc_to_target` should be left undefined. This will fetch the template of the dataset defined in the YAML file. + +For example, For Super Glue BoolQ, if we want to use the prompt template `GPT-3 Style` we can add this to the YAML file. + +```yaml +use_prompt: "promptsource:GPT-3 Style" +``` + +If you would like to run evaluation on all prompt templates, you can simply call it this way. + +```yaml +use_prompt: "promptsource:*" +``` + +### Setting metrics + +You're almost done! Now we need to choose how to score our task. + +- *If this is a multiple choice task:* do you just want to check your model's accuracy in choosing the correct answer choice? +- *If this is a generation task:* do you just want to check how often your model outputs *exactly the ground-truth output string provided*? + +If the answer to the above is no: you'll need to record what scoring metrics to use! Metrics can be listed in the following format: + +```yaml +metric_list: + - metric: + aggregation: + higher_is_better: + - metric: !function script.function + aggregation: ... + higher_is_better: ... +``` + +`aggregation` and `higher_is_better` can optionally be left out to default to the manually-set defaults if using a natively supported metric, otherwise it must be defined explicitly (for example, when using a custom metric implemented as a function). + +For a full list of natively supported metrics and aggregation functions see [`docs/task_guide.md`](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/task_guide.md). All metrics supported in [HuggingFace Evaluate](https://github.com/huggingface/evaluate/tree/main/metrics) can also be used, and will be loaded if a given metric name is not one natively supported in `lm-eval` or `hf_evaluate` is set to `true`. + +### Optional, More Advanced Setup + +Some tasks may require more advanced processing logic than is described in this guide. + +As a heuristic check: + +- Does your task require generating multiple free-form outputs per input document? +- Does your task require complex, multi-step post-processing of generated model outputs? +- Does your task require subsetting documents on the fly based on their content? +- Do you expect to compute metrics after applying multiple such processing steps on your model outputs? +- Does your task rely on metrics that need a custom implementation? + +For more detail on the task system and advanced features, see [`docs/task_guide.md`](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/task_guide.md). If none of the above sounds like they apply to your task, it's time to continue onto checking your task performance! + +### Task name + tags (registering a task) + +To test a task conveniently, it helps to *register* the task--that is, to give it a name and make the `lm-eval` library aware it exists! + +If you're writing your YAML file inside the `lm_eval/tasks` folder, you just need to give your task a name! You can do this inside your YAML file: + +```yaml +task: +``` + +Including a task name is mandatory. + +It is often also convenient to label your task with several `tag` values, though this field is optional: + +```yaml +tag: + - tag1 + - tag2 +``` + +This will add your task to the `tag1` and `tag2` tags, enabling people to know how to categorize your task, and if desired run all tasks in one of these groups at once, your task along with them. + +If your task is not in the `lm_eval/tasks` folder, you'll need to tell the Eval Harness where to look for YAML files. + +You can do this via the `--include_path` argument in `__main__.py`. This command will be used to initialize the `TaskManager` object which you can also use for your custom scripts. + +```python +task_manager = TaskManager(args.verbosity, include_path=args.include_path) +``` + +Passing `--tasks /path/to/yaml/file` is also accepted. + +### Advanced Group Configs + +While `tag` values are helpful when you want to be able to quickly and conveniently run a set of related tasks via `--tasks my_tag_name`, often, we wish to implement more complex logic. For example, the MMLU benchmark contains 57 *subtasks* that must all be *averaged* together in order to report a final 'MMLU score'. + +Groupings of tasks might also use particular variants of a task--for example, we might want to default to evaluating a task as 5-shot when called as part of a given grouping, but not have a preference for number of shots when evaluating it as a standalone. + +We implement this via **groups**, which are distinct from tags. Groups can be implemented via *group config* YAML files, which are laid out similarly but slightly differently to tasks' YAML configs. + +The most basic form of group can be defined via a YAML config similar to the following: + +```yaml +group: nli_tasks +task: + - cb + - anli_r1 + - rte +metadata: + version: 1.0 +``` + +This will behave almost identically to a `tag` that includes these 3 tasks, but with one key distinction: we'll print the `nli_tasks` group as a row (with no associated metrics) in our table of outputs, and visually show that these 3 tasks appear under its subheader. + +Now, let's assume we actually want to report an aggregate score for `nli_tasks`. We would instead use a YAML config like the following: + +```yaml +group: nli_tasks +task: + - cb + - anli_r1 + - rte +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true # defaults to `true`. Set this to `false` to do a "macro" average (taking each subtask's average accuracy, and summing those accuracies and dividing by 3)--by default we do a "micro" average (retain all subtasks' per-document accuracies, and take the mean over all documents' accuracies to get our aggregate mean). +metadata: + version: 1.0 +``` + +Similar to our `metric_list` for listing out the metrics we want to calculate for a given task, we use an `aggregate_metric_list` field to specify which metric name to aggregate across subtasks, what aggregation function to use, and whether we should micro- or macro- average these metrics. See [./task_guide.md](./task_guide.md) for a full list of related sub-keys. + +**[!Tip]: currently, we predominantly only support the aggregation of group metrics that use `mean` (either micro- or macro- averaged) over their subtasks. If you require even more complex aggregation rules, you may want to perform aggregation offline.** + +Group configs can be fairly complex! We can do various operations, such as defining new subtask(s) inline in our group YAML, overriding an existing task's specific config value, or nesting existing groups within our + +For example, let's build a config for evaluating MMLU and a few natural language inference tasks. For MMLU, we can write the name for the benchmark as a subtask written under `task`. You can configure the parameters such as `num_fewshot`. If the task being configured is a group such as `mmlu` or `super_glue`, the parameter set will be applied to all of the subtasks. + +```yaml +group: nli_and_mmlu +task: + - group: nli_tasks + task: + - cb + - anli_r1 + - rte + aggregate_metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - task: mmlu + num_fewshot: 2 +``` + +### Configuring python classes + +There can be occasions when yaml-based tasks cannot accommodate how a task is handled. LM-Eval supports the manually implementing tasks as was previously done before `0.4.x`. To register the task, you can simply make a yaml with the name of the task in `task` and the class object in `class` using the `!function` prefix. + +```yaml +task: squadv2 +class: !function task.SQuAD2 +``` + +This also applies to building group configurations with subtasks that are python classes. + +```yaml +group: scrolls +task: + - task: scrolls_qasper + class: !function task.Qasper + - task: scrolls_quality + class: !function task.QuALITY + - task: scrolls_narrativeqa + class: !function task.NarrativeQA + ... +``` + +You can also pass a custom argument to your class by accepting `config` in the custom class constructor. +Here's how to do it: + +```yaml +task: 20_newsgroups +class: !function task.Unitxt +recipe: card=cards.20_newsgroups,template=templates.classification.multi_class.title +``` + +In this example, `recipe` is the custom argument for the `Unitxt` class. + +## Beautifying Table Display + +To avoid conflict, each task needs to be registered with a unique name. Because of this, slight variations of task are still counted as unique tasks and need to be named uniquely. This could be done by appending an additional naming that may refer to the variation such as in MMLU where the template used to evaluated for flan are differentiated from the default by the prefix `mmlu_flan_*`. Printing the full task names can easily clutter the results table at the end of the evaluation especially when you have a long list of tasks or are using a benchmark that comprises of many tasks. To make it more legible, you can use `task_alias` and `group_alias` to provide an alternative task name and group name that will be printed. For example in `mmlu_abstract_algebra.yaml` we set `task_alias` to `abstract_algebra`. In group configs, a `group_alias` for a group can also be set. + +```yaml +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"include": "_default_template_yaml" +"task": "mmlu_abstract_algebra" +"task_alias": "abstract_algebra" +``` + +## Checking validity + +After registering your task, you can now check on your data downloading and verify that the few-shot samples look as intended. Run the following command with your desired args: + +```bash +python -m scripts.write_out \ + --output_base_path \ + --tasks \ + --sets \ + --num_fewshot K \ + --num_examples N \ +``` + +Open the file specified at the `--output_base_path ` and ensure it passes +a simple eye test. + +## Versioning + +One key feature in LM Evaluation Harness is the ability to version tasks and groups--that is, mark them with a specific version number that can be bumped whenever a breaking change is made. + +This version info can be provided by adding the following to your new task or group config file: + +```yaml +metadata: + version: 0 +``` + +Now, whenever a change needs to be made to your task in the future, please increase the version number by 1 so that users can differentiate the different task iterations and versions. + +If you are incrementing a task's version, please also consider adding a changelog to the task's README.md noting the date, PR number, what version you have updated to, and a one-liner describing the change. + +for example, + +- \[Dec 25, 2023\] (PR #999) Version 0.0 -> 1.0: Fixed a bug with answer extraction that led to underestimated performance. + +## Checking performance + equivalence + +It's now time to check models' performance on your task! In the evaluation harness, we intend to support a wide range of evaluation tasks and setups, but prioritize the inclusion of already-proven benchmarks following the precise evaluation setups in the literature where possible. + +To enable this, we provide a checklist that should be completed when contributing a new task, to enable accurate book-keeping and to ensure that tasks added to the library are well-tested and, where applicable, precedented. + +### Task Validity Checklist + +The checklist is the following: + +For adding novel benchmarks/datasets to the library: + +- [ ] Is the task an existing benchmark in the literature? + - [ ] Have you referenced the original paper that introduced the task? + - [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + +If other tasks on this dataset are already supported: + +- [ ] Is the "Main" variant of this task clearly denoted? +- [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +- [ ] Have you noted which, if any, published evaluation setups are matched by this variant? + +It is recommended to include a filled-out copy of this checklist in the README.md for the subfolder you are creating, if you have created a new subfolder in `lm_eval/tasks`. + +**Finally, please add a short description of your task(s), along with a link to its subfolder in lm_eval/tasks, to [`lm_eval/tasks/README.md`](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/README.md) so that users can discover your task in the library, and follow the link to your README for more information about the variants supported, their task names, and the original source of the dataset and/or evaluation setup.** + +## Submitting your task + +You're all set! Now push your work and make a pull request to the `main` branch! Thanks for the contribution :). If there are any questions, please leave a message in the `#lm-thunderdome` channel on the EAI discord! diff --git a/lm-evaluation-harness/docs/task_guide.md b/lm-evaluation-harness/docs/task_guide.md new file mode 100644 index 0000000000000000000000000000000000000000..61172788f96865300020c3ed3269627fc8a60191 --- /dev/null +++ b/lm-evaluation-harness/docs/task_guide.md @@ -0,0 +1,335 @@ +# Task Configuration + +The `lm-evaluation-harness` is meant to be an extensible and flexible framework within which many different evaluation tasks can be defined. All tasks in the new version of the harness are built around a YAML configuration file format. + +These YAML configuration files, along with the current codebase commit hash, are intended to be shareable such that providing the YAML config enables another researcher to precisely replicate the evaluation setup used by another, in the case that the prompt or setup differs from standard `lm-eval` task implementations. + +While adding a standard evaluation task on a new dataset can be occasionally as simple as swapping out a Hugging Face dataset path in an existing file, more specialized evaluation setups also exist. Here we'll provide a crash course on the more advanced logic implementable in YAML form available to users. + +If your intended task relies on features beyond what is described in this guide, we'd love to hear about it! Feel free to open an issue describing the scenario on Github, create a PR to the project with a proposed implementation, or ask in the `#lm-thunderdome` channel on the EleutherAI discord. + +## Configurations + +Tasks are configured via the `TaskConfig` object. Below, we describe all fields usable within the object, and their role in defining a task. + +### Parameters + +Task naming + registration: + +- **task** (`str`, defaults to None) โ€” name of the task. +- **task_alias** (`str`, defaults to None) - Alias of the task name that will be printed in the final table results. +- **tag** (`str`, *optional*) โ€” name of the task tags(s) a task belongs to. Enables one to run all tasks with a specified tag name at once. + +Dataset configuration options: + +- **dataset_path** (`str`) โ€” The name of the dataset as listed by HF in the datasets Hub. +- **dataset_name** (`str`, *optional*, defaults to None) โ€” The name of what HF calls a โ€œdata instanceโ€ or sub-task of the benchmark. If your task does not contain any data instances, just leave this to default to None. (If you're familiar with the HF `datasets.load_dataset` function, these are just the first 2 arguments to it.) +- **dataset_kwargs** (`dict`, *optional*) โ€” Auxiliary arguments that `datasets.load_dataset` accepts. This can be used to specify arguments such as `data_files` or `data_dir` if you want to use local datafiles such as json or csv. +- **custom_dataset** (`Callable`, *optional) - A function that returns a `dict[str, datasets.Dataset]` (, dataset) object. This can be used to load a dataset from a custom source or to preprocess the dataset in a way that is not supported by the `datasets` library. Will have access to `metadata` field if defined (from config and passed to TaskManager), and `model_args` from runtime (if using `evaluate`). +- **training_split** (`str`, *optional*) โ€” Split in the dataset to use as the training split. +- **validation_split** (`str`, *optional*) โ€” Split in the dataset to use as the validation split. +- **test_split** (`str`, *optional*) โ€” Split in the dataset to use as the test split. +- **fewshot_split** (`str`, *optional*) โ€” Split in the dataset to draw few-shot exemplars from. assert that this not None if num_fewshot > 0. +- **process_docs** (`Callable`, *optional*) โ€” Optionally define a function to apply to each HF dataset split, to preprocess all documents before being fed into prompt template rendering or other evaluation steps. Can be used to rename dataset columns, or to process documents into a format closer to the expected format expected by a prompt template. + +Prompting / in-context formatting options: + +- **use_prompt** (`str`, *optional*) โ€” Name of prompt in promptsource to use. if defined, will overwrite doc_to_text, doc_to_target, and doc_to_choice. +- **description** (`str`, *optional*) โ€” An optional prepended Jinja2 template or string which will be prepended to the few-shot examples passed into the model, often describing the task or providing instructions to a model, such as `"The following are questions (with answers) about {{subject}}.\n\n"`. No delimiters or spacing are inserted between the description and the first few-shot example. +- **doc_to_text** (`Union[Callable, str]`, *optional*) โ€” Jinja2 template, string, or function to process a sample into the appropriate input for the model. +- **doc_to_target** (`Union[Callable, str]`, *optional*) โ€” Jinja2 template, string, or function to process a sample into the appropriate target output for the model. For multiple choice tasks, this should return an index into the answer choice list of the correct answer. +- **doc_to_choice** (`Union[Callable, str]`, *optional*) โ€” Jinja2 template, string, or function to process a sample into a list of possible string choices for `multiple_choice` tasks. Left undefined for `generate_until` tasks. +- **fewshot_delimiter** (`str`, *optional*, defaults to "\n\n") โ€” String to insert between few-shot examples. +- **target_delimiter** (`str`, *optional*, defaults to `" "`) โ€” String to insert between input and target output for the datapoint being tested. +- **gen_prefix** (`str`, *optional*) โ€” String to append after the <|assistant|> token. For example, if the task is to generate a question, the gen_prefix could be "The answer is: " to prompt the model to generate an answer to the question. If not using a chat template then this string will be appended to the end of the prompt. + +Runtime configuration options: + +- **num_fewshot** (`int`, *optional*, defaults to 0) โ€” Number of few-shot examples before the input. +- **batch_size** (`int`, *optional*, defaults to 1) โ€” Batch size. + +Scoring details: + +- **metric_list** (`str`, *optional*, defaults to None) โ€” A list of metrics to use for evaluation. See docs for expected format. +- **output_type** (`str`, *optional*, defaults to "generate_until") โ€” Selects the type of model output for the given task. Options are `generate_until`, `loglikelihood`, `loglikelihood_rolling`, and `multiple_choice`. +- **generation_kwargs** (`dict`, *optional*) โ€” Auxiliary arguments for the `generate` function from HF transformers library. Advanced keyword arguments may not be supported for non-HF LM classes. +- **repeats** (`int`, *optional*, defaults to 1) โ€” Number of repeated runs through model for each sample. Can be used for cases such as self-consistency. +- **filter_list** (`Union[str, list]`, *optional*) โ€” List of filters to postprocess model outputs. See below for further detail on the filter API. +- **should_decontaminate** (`bool`, *optional*, defaults to False) - Whether to decontaminate or not. +- **doc_to_decontamination_query** (`str`, *optional*) โ€” Query for decontamination if `should_decontaminate` is True. If `should_decontaminate` is True but `doc_to_decontamination_query` is `None`, `doc_to_decontamination_query` will follow `doc_to_text`. + +Other: + +- **metadata** (`dict`, *optional*) โ€” An optional field where arbitrary metadata can be passed. Most tasks should include a `version` key in this field that is used to denote the version of the yaml config. Other special metadata keys are: `num_fewshot`, to override the printed `n-shot` table column for a task. Will also be passed to the `custom_dataset` function if defined. + +## Filters + +A key component of the `lm-evaluation-harness` library is the `Filter` object. In a typical evaluation run of the harness, we take the formatted inputs and run them through our LM, with the appropriate output type (greedy or free-form generation, or loglikelihood-based comparative scoring). + +After getting scores or output text from our LM on each `Instance` or document in the dataset, we then need to feed these responses into a metric or scoring function to return scores to a user. + +However, certain tasks may require more complex behavior than directly turning over model outputs to a metric function. For example, we may want to post-process our output text by truncating it or extracting a model's answer, we may want to ensemble over multiple "takes" on a different document, et cetera. + +**Detailed Aside**: +We do such post-processing by operating on *responses*, which are stored after running an LM on an `Instance` from the task in `Instance.resps`. + +`resps` is a `List[str]` for each instance, and we pass a `List[List[]]` to our filters that is a list of `[instance.resps for instance in instances]`. + +Our filters, after completing a pipeline, must return a `List[]` which we then unpack and store each element of in `Instance.filtered_resps` for the corresponding instance. Thus, we take as input a list of returns from our model for each doc, and must return a return from our model *without it being wrapped in a list* for each doc. +**End Aside** + +A full list of supported filter operations can be found in `lm_eval/filters/__init__.py`. Contributions of new filter types are welcome! + +### Multiple Filter Pipelines + +Tasks need not be limited to a single filter pipeline. We enable users to run multiple, distinct, filter pipelines on *the same model outputs* generated in one run on a task. + +As a case study, let's look at an implementation of solving the Gsm8k math word problem benchmark in `lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml`. Here, we are emulating the setup used by [Self-Consistency Improves Chain of Thought Prompting](https://arxiv.org/abs/2203.11171), in which evaluation is performed by generating N chain-of-thought outputs from a model via temperature-based sampling, then selecting the answers output by the model at the end of the chains of thought, then majority voting across all those numeric answers. + +Within our YAML file: + +```yaml +... +repeats: 64 +filter_list: + - name: "score-first" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]*[0-9]+)" + - function: "take_first" + - name: "maj@64" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]*[0-9]+)" + - function: "majority_vote" + - function: "take_first" + - name: "maj@8" + filter: + - function: "take_first_k" + k: 8 + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]*[0-9]+)" + - function: "majority_vote" + - function: "take_first" +``` + +We are able to provide multiple different filter pipelines, each with their own name and list of filters to apply in sequence. + +Our first filter pipeline implements + +- applying a regex to the model generations (extracting the number within the phrase "The answer is (number)") +- selecting only the first out of the 64 model answers + +Then scoring this single answer. + +```yaml +- name: "score-first" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]*[0-9]+)" + - function: "take_first" +``` + +Our second filter pipeline, "maj@64", does majority voting across all 64 answers via: + +- applying the same regex to all responses, to get the numerical answer from the model for each of the 64 responses per problem +- applying majority voting to all responses, which then returns a length-1 `[]` list for each +- taking the first element of this length-1 list, to then score the sole response `` for each document. + +```yaml +- name: "maj@64" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]*[0-9]+)" + - function: "majority_vote" + - function: "take_first" +``` + +Our final filter pipeline, "maj@8", does majority voting across the first 8 of the model's responses per document via: + +- subsetting the len-64 list of responses `[answer1, answer2, ..., answer64]` to `[answer1, answer2, ..., answer8]` for each document +- performing the same sequence of filters on these new sets of 8 responses, for each document. + +```yaml +- name: "maj@8" + filter: + - function: "take_first_k" + k: 8 + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]*[0-9]+)" + - function: "majority_vote" + - function: "take_first" +``` + +Thus, given the 64 responses from our LM on each document, we can report metrics on these responses in these 3 different ways, as defined by our filter pipelines. + +### Adding a custom filter + +Just like adding a custom model with `register_model` decorator one is able to do the same with filters, for example + +```python +from lm_eval.api.filter import Filter +from lm_eval.api.registry import register_filter + +@register_filter("new_filter") +class NewFilter(Filter) + ... +``` + +## Embedded Python Code + +Use can use python functions for certain arguments by using the `!function` operator after the argument name followed by `.`. This feature can be used for the following arguments: + +1. `doc_to_text` +2. `doc_to_target` +3. `doc_to_choice` +4. `aggregation` for a `metric` in `metric_list` + +## (No Longer Recommended) Direct `Task` Subclassing + +The prior implementation method of new tasks was to subclass `Task`. While we intend to migrate all tasks to the new YAML implementation option going forward, it remains possible to subclass the Task class and implement custom logic. For more information, see `docs/task_guide.md` in v0.3.0 of the `lm-evaluation-harness`. + +## Including a Base YAML + +You can base a YAML on another YAML file as a template. This can be handy when you need to just change the prompt for `doc_to_text` but keep the rest the same or change `filters` to compare which is better. Simply use `include` in the YAML file and write the name of the template you want to base from. This assumes that the base template is in the same directory. Otherwise, You will need to define the full path. + +```yaml +include: +... +``` + +You can find an example of how to use this feature at [gsm8k-cot-self-consistency.yaml](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml) where it is based off [gsm8k-cot.yaml](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/gsm8k/gsm8k-cot.yaml) + +## Passing Arguments to Metrics + +Metrics can be defined in the `metric_list` argument when building the YAML config. Multiple metrics can be listed along with any auxiliary arguments. For example, setting the [`exact_match` metric](https://github.com/huggingface/evaluate/tree/main/metrics/exact_match), auxiliary arguments such as `ignore_case`, `ignore_punctuation`, `regexes_to_ignore` can be listed as well. They will be added to the metric function as `kwargs`. Some metrics have predefined values for `aggregation` and `higher_is_better` so listing the metric name only can be sufficient. + +```yaml +metric_list: + - metric: acc + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + regexes_to_ignore: + - "," + - "\\$" +``` + +### Natively Supported Metrics + +Here we list all metrics currently supported natively in `lm-eval`: + +Metrics: + +- `acc` (accuracy) +- `acc_norm` (length-normalized accuracy) +- `acc_mutual_info` (baseline loglikelihood - normalized accuracy) +- `perplexity` +- `word_perplexity` (perplexity per word) +- `byte_perplexity` (perplexity per byte) +- `bits_per_byte` +- `matthews_corrcoef` (Matthews correlation coefficient) +- `f1` (F1 score) +- `bleu` +- `chrf` +- `ter` + +Aggregation functions: + +- `mean` +- `median` +- `perplexity` +- `weighted_perplexity` +- `bits_per_byte` + +### Adding a Multiple Choice Metric + +Adding a multiple choice metric has a few steps. To get it working you need to: + +1. register a metric function +2. register an aggregation function +3. update the `Task` definition to make sure the correct arguments are passed + +The default metric and aggregation functions are in `lm_eval/api/metrics.py`, and you can add a function there if it's for general use. The metrics are towards the bottom of the file and look like this: + +```python +@register_metric( + metric="mcc", + higher_is_better=True, + output_type="multiple_choice", + aggregation="matthews_corrcoef", +) +def mcc_fn(items): # This is a passthrough function + return items +``` + +Note that many of these are passthrough functions, and for multiple choice (at least) this function is never actually called. + +Aggregation functions are defined towards the top of the file, here's an example: + +```python +@register_aggregation("matthews_corrcoef") +def matthews_corrcoef(items): + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + return sklearn.metrics.matthews_corrcoef(golds, preds) +``` + +This function returns a single numeric value. The input is defined in `Task.process_results` in `lm_eval/api/task.py`. There's a section that looks like this: + +```python +result_dict = { + **({"acc": acc} if "acc" in use_metric else {}), + **({"f1": (gold, pred)} if "f1" in use_metric else {}), + **({"mcc": (gold, pred)} if "mcc" in use_metric else {}), + **({"acc_norm": acc_norm} if "acc_norm" in use_metric else {}), + **({"exact_match": exact_match} if "exact_match" in use_metric else {}), +} +``` + +The value here determines the input to the aggregation function, though the name used matches the metric function. These metrics all have simple needs and just need the accuracy or gold and predicted values, but immediately below this there are examples of metrics with more complicated needs you can use as reference. + +## Good Reference Tasks + +Contributing a new task can be daunting! Luckily, much of the work has often been done for you in a different, similarly evaluated task. Good examples of task implementations to study include: + +Multiple choice tasks: + +- SciQ (`lm_eval/tasks/sciq/sciq.yaml`) + +Corpus perplexity evaluations: + +- Wikitext (`lm_eval/tasks/wikitext/wikitext.yaml`) + +Generative tasks: + +- GSM8k (`lm_eval/tasks/gsm8k/gsm8k.yaml`) + +Tasks using complex filtering: + +- GSM8k with CoT (+ with Self-Consistency): (`lm_eval/tasks/gsm8k/gsm8k-cot.yaml` ; `lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml`) + +# Group Configuration + +When evaluating a language model, it is not unusual to test across a number of tasks that may not be related to one another in order to assess a variety of capabilities. To this end, it may be cumbersome to have to list the set of tasks or add a new group name to each yaml of each individual task. + +To solve this, we can create a **group** yaml config. This is a config that contains the names of the tasks that should be included in a particular group. The config consists of two main keys: a `group` key which denotes the name of the group (as it would be called from the command line, e.g. `mmlu`) and a `task` key which is where we can list the tasks. The tasks listed in `task` are the task names that have been registered. A good example of a group yaml config can be found at [../lm_eval/tasks/mmlu/default/_mmlu.yaml]. See also the [New Task Guide](./new_task_guide.md) for a more in-depth and tutorial-esque explanation of how to write complex GroupConfigs. + +## Configurations + +Groups are configured via the `GroupConfig` object. Below, we describe all fields usable within the object, and their role in defining a task. + +### Parameters + +- **group** (`str`, defaults to `None`) โ€” name of the group. Used to invoke it from the command line. +- **group_alias** (`str`, defaults to `None`) - Alternative name for the group that will be printed in the table output. +- **task** (`Union[str, list]`, defaults to `None`) - List of tasks that constitute the group. +- **aggregate_metric_list** (`list`, defaults to `None`) - similar to `metric_list` in TaskConfigs, provide a list of configurations for metrics that should be aggregated across subtasks. Leaving empty will result in no aggregation being performed for this group. Keys for each list entry are: + - `metric: str` - the name of the metric to aggregate over (all subtasks must report a metric holding this name.) + - `aggregation: str` - what aggregation function to apply to aggregate these per-subtask metrics. **currently, only `mean` is supported.** + - `weight_by_size: bool = True` whether to perform micro- averaging (`True`) or macro- (`False`) averaging of subtasks' accuracy scores when reporting the group's metric. MMLU, for example, averages over per-document accuracies (the *micro average*), resulting in the same accuracy as if one simply concatenated all 57 subjects into a single dataset and evaluated accuracy on that dataset. + - `filter_list: Union[str, List[str]] = "none"` - what filter keys one should match on to aggregate results. For example, if trying to aggregate over the `exact_match` metric using `strict-match` filter for `bbh_cot_zeroshot`, then set this to be `filter_list: "strict-match"`. +- **metadata** (`dict`, *optional*) - As with TaskConfigs, a field where extra config metadata can be passed. set the `num_fewshot` key within this to override the printed n_shot value in a results table for your group, for example. diff --git a/lm-evaluation-harness/eval.log b/lm-evaluation-harness/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..4831d29b6b8ad07f7827eb478c091d32a615ca91 --- /dev/null +++ b/lm-evaluation-harness/eval.log @@ -0,0 +1,2265 @@ +nohup: ignoring input +2025-06-20:18:36:55 INFO [__main__:440] Selected Tasks: ['triviaqa'] +2025-06-20:18:36:55 INFO [evaluator:189] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234 +2025-06-20:18:36:55 INFO [evaluator:227] Initializing hf model, with arguments: {'pretrained': '/root/chenxr/models/Llama-2-7b-hf'} +2025-06-20:18:36:55 INFO [models.huggingface:137] Using device 'cuda:0' +2025-06-20:18:36:56 INFO [models.huggingface:382] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'} + Loading checkpoint shards: 0%| | 0/2 [00:00=0.21.0 (from lm-eval==1.0.0)\n", + " Downloading accelerate-0.24.1-py3-none-any.whl (261 kB)\n", + "\u001b[2K \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting evaluate (from lm-eval==1.0.0)\n", + " Downloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n", + "\u001b[2K \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting datasets>=2.0.0 (from lm-eval==1.0.0)\n", + " Downloading datasets-2.15.0-py3-none-any.whl (521 kB)\n", + "\u001b[2K \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jsonlines (from lm-eval==1.0.0)\n", + " Downloading jsonlines-4.0.0-py3-none-any.whl (8.7 kB)\n", + "Requirement already satisfied: numexpr in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (2.8.7)\n", + "Collecting peft>=0.2.0 (from lm-eval==1.0.0)\n", + " Downloading peft-0.6.2-py3-none-any.whl (174 kB)\n", + "\u001b[2K \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m174.7/174.7 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pybind11>=2.6.2 (from lm-eval==1.0.0)\n", + " Downloading pybind11-2.11.1-py3-none-any.whl (227 kB)\n", + "\u001b[2K \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m227.7/227.7 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pytablewriter (from lm-eval==1.0.0)\n", + " Downloading pytablewriter-1.2.0-py3-none-any.whl (111 kB)\n", + "\u001b[2K \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m111.1/111.1 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting rouge-score>=0.0.4 (from lm-eval==1.0.0)\n", + " Downloading rouge_score-0.1.2.tar.gz (17 kB)\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Collecting sacrebleu>=1.5.0 (from lm-eval==1.0.0)\n", + " Downloading sacrebleu-2.3.2-py3-none-any.whl (119 kB)\n", + "\u001b[2K \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m119.7/119.7 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: scikit-learn>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (1.2.2)\n", + "Collecting sqlitedict (from lm-eval==1.0.0)\n", + " Downloading sqlitedict-2.1.0.tar.gz (21 kB)\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: torch>=1.8 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (2.1.0+cu118)\n", + "Collecting tqdm-multiprocess (from lm-eval==1.0.0)\n", + " Downloading tqdm_multiprocess-0.0.11-py3-none-any.whl (9.8 kB)\n", + "Requirement already satisfied: transformers>=4.1 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (4.35.2)\n", + "Collecting zstandard (from lm-eval==1.0.0)\n", + " Downloading zstandard-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n", + "\u001b[2K \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m29.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (1.23.5)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (23.2)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (5.9.5)\n", + "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (6.0.1)\n", + "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (0.19.4)\n", + "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (9.0.0)\n", + "Collecting pyarrow-hotfix (from datasets>=2.0.0->lm-eval==1.0.0)\n", + " Downloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n", + "Collecting dill<0.3.8,>=0.3.0 (from datasets>=2.0.0->lm-eval==1.0.0)\n", + " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n", + "\u001b[2K \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (1.5.3)\n", + "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (4.66.1)\n", + "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (3.4.1)\n", + "Collecting multiprocess (from datasets>=2.0.0->lm-eval==1.0.0)\n", + " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n", + "\u001b[2K \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (2023.6.0)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (3.8.6)\n", + "Collecting responses<0.19 (from evaluate->lm-eval==1.0.0)\n", + " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n", + "Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from peft>=0.2.0->lm-eval==1.0.0) (0.4.0)\n", + "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm-eval==1.0.0) (1.4.0)\n", + "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm-eval==1.0.0) (3.8.1)\n", + "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm-eval==1.0.0) (1.16.0)\n", + "Collecting portalocker (from sacrebleu>=1.5.0->lm-eval==1.0.0)\n", + " Downloading portalocker-2.8.2-py3-none-any.whl (17 kB)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==1.0.0) (2023.6.3)\n", + "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==1.0.0) (0.9.0)\n", + "Collecting colorama (from sacrebleu>=1.5.0->lm-eval==1.0.0)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==1.0.0) (4.9.3)\n", + "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm-eval==1.0.0) (1.11.3)\n", + "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm-eval==1.0.0) (1.3.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm-eval==1.0.0) (3.2.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (3.13.1)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (4.5.0)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (3.2.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (3.1.2)\n", + "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (2.1.0)\n", + "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.1->lm-eval==1.0.0) (0.15.0)\n", + "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonlines->lm-eval==1.0.0) (23.1.0)\n", + "Requirement already satisfied: setuptools>=38.3.0 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm-eval==1.0.0) (67.7.2)\n", + "Collecting DataProperty<2,>=1.0.1 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading DataProperty-1.0.1-py3-none-any.whl (27 kB)\n", + "Collecting mbstrdecoder<2,>=1.0.0 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading mbstrdecoder-1.1.3-py3-none-any.whl (7.8 kB)\n", + "Collecting pathvalidate<4,>=2.3.0 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading pathvalidate-3.2.0-py3-none-any.whl (23 kB)\n", + "Collecting tabledata<2,>=1.3.1 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading tabledata-1.3.3-py3-none-any.whl (11 kB)\n", + "Collecting tcolorpy<1,>=0.0.5 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading tcolorpy-0.1.4-py3-none-any.whl (7.9 kB)\n", + "Collecting typepy[datetime]<2,>=1.3.2 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading typepy-1.3.2-py3-none-any.whl (31 kB)\n", + "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (3.3.2)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (6.0.4)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (4.0.3)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (1.9.2)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (1.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (1.3.1)\n", + "Requirement already satisfied: chardet<6,>=3.0.4 in /usr/local/lib/python3.10/dist-packages (from mbstrdecoder<2,>=1.0.0->pytablewriter->lm-eval==1.0.0) (5.2.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.0.0->lm-eval==1.0.0) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.0.0->lm-eval==1.0.0) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.0.0->lm-eval==1.0.0) (2023.7.22)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.8.0 in /usr/local/lib/python3.10/dist-packages (from typepy[datetime]<2,>=1.3.2->pytablewriter->lm-eval==1.0.0) (2.8.2)\n", + "Requirement already satisfied: pytz>=2018.9 in /usr/local/lib/python3.10/dist-packages (from typepy[datetime]<2,>=1.3.2->pytablewriter->lm-eval==1.0.0) (2023.3.post1)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.8->lm-eval==1.0.0) (2.1.3)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score>=0.0.4->lm-eval==1.0.0) (8.1.7)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8->lm-eval==1.0.0) (1.3.0)\n", + "Building wheels for collected packages: lm-eval, rouge-score, sqlitedict\n", + " Building wheel for lm-eval (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for lm-eval: filename=lm_eval-1.0.0-py3-none-any.whl size=994254 sha256=88356155b19f2891981ecef948326ad6ce8ca40a6009378410ec20d0e225995a\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-9v6ye7h3/wheels/17/01/26/599c0779e9858a70a73fa8a306699b5b9a868f820c225457b0\n", + " Building wheel for rouge-score (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24933 sha256=6bb0d44e4881972c43ce194e7cb65233d309758cb15f0dec54590d3d2efcfc36\n", + " Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\n", + " Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for sqlitedict: filename=sqlitedict-2.1.0-py3-none-any.whl size=16863 sha256=5747f7dd73ddf3d8fbcebf51b5e4f718fabe1e94bccdf16d2f22a2e65ee7fdf4\n", + " Stored in directory: /root/.cache/pip/wheels/79/d6/e7/304e0e6cb2221022c26d8161f7c23cd4f259a9e41e8bbcfabd\n", + "Successfully built lm-eval rouge-score sqlitedict\n", + "Installing collected packages: sqlitedict, zstandard, tcolorpy, pybind11, pyarrow-hotfix, portalocker, pathvalidate, mbstrdecoder, jsonlines, dill, colorama, typepy, tqdm-multiprocess, sacrebleu, rouge-score, responses, multiprocess, accelerate, datasets, DataProperty, tabledata, peft, evaluate, pytablewriter, lm-eval\n", + "Successfully installed DataProperty-1.0.1 accelerate-0.24.1 colorama-0.4.6 datasets-2.15.0 dill-0.3.7 evaluate-0.4.1 jsonlines-4.0.0 lm-eval-1.0.0 mbstrdecoder-1.1.3 multiprocess-0.70.15 pathvalidate-3.2.0 peft-0.6.2 portalocker-2.8.2 pyarrow-hotfix-0.6 pybind11-2.11.1 pytablewriter-1.2.0 responses-0.18.0 rouge-score-0.1.2 sacrebleu-2.3.2 sqlitedict-2.1.0 tabledata-1.3.3 tcolorpy-0.1.4 tqdm-multiprocess-0.0.11 typepy-1.3.2 zstandard-0.22.0\n" + ] + } + ], + "source": [ + "# Install LM-Eval\n", + "!pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 0, + "referenced_widgets": [ + "a1d3a8aa016544a78e8821c8f6199e06", + "f61ed33fad754146bdd2ac9db1ba1c48", + "bfa0af6aeff344c6845e1080a878e92e", + "fd1ad9e0367d4004aae853b91c3a7617", + "6b2d90209ec14230b3d58a74ac9b83bf", + "a73f357065d34d7baf0453ae4a8d75e2", + "46f521b73fd943c081c648fd873ebc0a", + "7c5689bc13684db8a22681f41863dddd", + "48763b6233374554ae76035c0483066f", + "4986a21eb560448fa79f4b25cde48951", + "aed3acd2f2d74003b44079c333a0698e" + ] + }, + "id": "uyO5MaKkZyah", + "outputId": "d46e8096-5086-4e49-967e-ea33d4a2a335" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a1d3a8aa016544a78e8821c8f6199e06", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading builder script: 0%| | 0.00/5.67k [00:00\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "fthNg3ywO-kA" + }, + "outputs": [], + "source": [ + "YAML_cola_string = \"\"\"\n", + "tag: yes_or_no_tasks\n", + "task: demo_cola\n", + "dataset_path: glue\n", + "dataset_name: cola\n", + "output_type: multiple_choice\n", + "training_split: train\n", + "validation_split: validation\n", + "doc_to_text: \"{{sentence}}\\nQuestion: Does this sentence make sense?\\nAnswer:\"\n", + "doc_to_target: label\n", + "doc_to_choice: [\"no\", \"yes\"]\n", + "should_decontaminate: true\n", + "doc_to_decontamination_query: sentence\n", + "metric_list:\n", + " - metric: acc\n", + "\"\"\"\n", + "with open(\"cola.yaml\", \"w\") as f:\n", + " f.write(YAML_cola_string)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "XceRKCuuDtbn" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-29:11:56:33,016 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", + "2023-11-29 11:56:33.852995: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2023-11-29 11:56:33.853050: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2023-11-29 11:56:33.853087: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2023-11-29 11:56:35.129047: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "2023-11-29:11:56:38,546 INFO [__main__.py:132] Verbosity set to INFO\n", + "2023-11-29:11:56:47,509 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", + "2023-11-29:11:56:47,509 INFO [__main__.py:143] Including path: ./\n", + "2023-11-29:11:56:47,517 INFO [__main__.py:205] Selected Tasks: ['yes_or_no_tasks']\n", + "2023-11-29:11:56:47,520 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", + "2023-11-29:11:56:47,550 INFO [huggingface.py:120] Using device 'cuda'\n", + "2023-11-29:11:57:08,743 WARNING [task.py:614] [Task: demo_cola] metric acc is defined, but aggregation is not. using default aggregation=mean\n", + "2023-11-29:11:57:08,743 WARNING [task.py:626] [Task: demo_cola] metric acc is defined, but higher_is_better is not. using default higher_is_better=True\n", + "Downloading builder script: 100% 28.8k/28.8k [00:00<00:00, 52.7MB/s]\n", + "Downloading metadata: 100% 28.7k/28.7k [00:00<00:00, 51.9MB/s]\n", + "Downloading readme: 100% 27.9k/27.9k [00:00<00:00, 48.0MB/s]\n", + "Downloading data: 100% 377k/377k [00:00<00:00, 12.0MB/s]\n", + "Generating train split: 100% 8551/8551 [00:00<00:00, 19744.58 examples/s]\n", + "Generating validation split: 100% 1043/1043 [00:00<00:00, 27057.01 examples/s]\n", + "Generating test split: 100% 1063/1063 [00:00<00:00, 22705.17 examples/s]\n", + "2023-11-29:11:57:11,698 INFO [task.py:355] Building contexts for task on rank 0...\n", + "2023-11-29:11:57:11,704 INFO [evaluator.py:319] Running loglikelihood requests\n", + "100% 20/20 [00:03<00:00, 5.15it/s]\n", + "fatal: not a git repository (or any of the parent directories): .git\n", + "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", + "| Tasks |Version|Filter|n-shot|Metric|Value| |Stderr|\n", + "|---------------|-------|------|-----:|------|----:|---|-----:|\n", + "|yes_or_no_tasks|N/A |none | 0|acc | 0.7|ยฑ |0.1528|\n", + "| - demo_cola |Yaml |none | 0|acc | 0.7|ยฑ |0.1528|\n", + "\n", + "| Groups |Version|Filter|n-shot|Metric|Value| |Stderr|\n", + "|---------------|-------|------|-----:|------|----:|---|-----:|\n", + "|yes_or_no_tasks|N/A |none | 0|acc | 0.7|ยฑ |0.1528|\n", + "\n" + ] + } + ], + "source": [ + "# !accelerate launch --no_python\n", + "%env LOGLEVEL=DEBUG\n", + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", + " --include_path ./ \\\n", + " --tasks yes_or_no_tasks \\\n", + " --limit 10 \\\n", + " --output output/yes_or_no_tasks/ \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XceRKCuuDtbn" + }, + "source": [ + "## Edit Prompt Templates Quickly\n", + "\n", + "The following is a yaml made to evaluate the specific subtask of `high_school_geography` from MMLU. It uses the standard prompt where the we choose the letters from the options with most likelihood as the model's prediction." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "GTFvdt9kSlBG" + }, + "outputs": [], + "source": [ + "YAML_mmlu_geo_string = \"\"\"\n", + "task: demo_mmlu_high_school_geography\n", + "dataset_path: cais/mmlu\n", + "dataset_name: high_school_geography\n", + "description: \"The following are multiple choice questions (with answers) about high school geography.\\n\\n\"\n", + "test_split: test\n", + "fewshot_split: dev\n", + "fewshot_config:\n", + " sampler: first_n\n", + "output_type: multiple_choice\n", + "doc_to_text: \"{{question.strip()}}\\nA. {{choices[0]}}\\nB. {{choices[1]}}\\nC. {{choices[2]}}\\nD. {{choices[3]}}\\nAnswer:\"\n", + "doc_to_choice: [\"A\", \"B\", \"C\", \"D\"]\n", + "doc_to_target: answer\n", + "metric_list:\n", + " - metric: acc\n", + " aggregation: mean\n", + " higher_is_better: true\n", + " - metric: acc_norm\n", + " aggregation: mean\n", + " higher_is_better: true\n", + "\"\"\"\n", + "with open(\"mmlu_high_school_geography.yaml\", \"w\") as f:\n", + " f.write(YAML_mmlu_geo_string)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "jyKOfCsKb-xy" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-29:11:57:23,598 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", + "2023-11-29 11:57:24.719750: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2023-11-29 11:57:24.719806: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2023-11-29 11:57:24.719847: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2023-11-29 11:57:26.656125: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "2023-11-29:11:57:31,563 INFO [__main__.py:132] Verbosity set to INFO\n", + "2023-11-29:11:57:40,541 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", + "2023-11-29:11:57:40,541 INFO [__main__.py:143] Including path: ./\n", + "2023-11-29:11:57:40,558 INFO [__main__.py:205] Selected Tasks: ['demo_mmlu_high_school_geography']\n", + "2023-11-29:11:57:40,559 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", + "2023-11-29:11:57:40,589 INFO [huggingface.py:120] Using device 'cuda'\n", + "Downloading builder script: 100% 5.84k/5.84k [00:00<00:00, 17.7MB/s]\n", + "Downloading metadata: 100% 106k/106k [00:00<00:00, 892kB/s] \n", + "Downloading readme: 100% 39.7k/39.7k [00:00<00:00, 631kB/s]\n", + "Downloading data: 100% 166M/166M [00:01<00:00, 89.0MB/s]\n", + "Generating auxiliary_train split: 100% 99842/99842 [00:07<00:00, 12536.83 examples/s]\n", + "Generating test split: 100% 198/198 [00:00<00:00, 1439.20 examples/s]\n", + "Generating validation split: 100% 22/22 [00:00<00:00, 4181.76 examples/s]\n", + "Generating dev split: 100% 5/5 [00:00<00:00, 36.25 examples/s]\n", + "2023-11-29:11:58:09,798 INFO [task.py:355] Building contexts for task on rank 0...\n", + "2023-11-29:11:58:09,822 INFO [evaluator.py:319] Running loglikelihood requests\n", + "100% 40/40 [00:05<00:00, 7.86it/s]\n", + "fatal: not a git repository (or any of the parent directories): .git\n", + "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", + "| Tasks |Version|Filter|n-shot| Metric |Value| |Stderr|\n", + "|-------------------------------|-------|------|-----:|--------|----:|---|-----:|\n", + "|demo_mmlu_high_school_geography|Yaml |none | 0|acc | 0.3|ยฑ |0.1528|\n", + "| | |none | 0|acc_norm| 0.3|ยฑ |0.1528|\n", + "\n" + ] + } + ], + "source": [ + "# !accelerate launch --no_python\n", + "%env LOGLEVEL=DEBUG\n", + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", + " --include_path ./ \\\n", + " --tasks demo_mmlu_high_school_geography \\\n", + " --limit 10 \\\n", + " --output output/mmlu_high_school_geography/ \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jyKOfCsKb-xy" + }, + "source": [ + "We could also evaluate this task in a different way. For example, instead of observing the loglikelihood of the letters, we can instead evaluate on the choices themselves as the continuation. This is done by simply changing `doc_to_choice` from a list of letters to the corresponding `choices` field from the HF dataset. We write `\"{{choices}}\"` so that the string field is interpreted as jinja string that acquires the list from the HF dataset directly.\n", + "\n", + "Another convenient feature here is since we're only modifying the `doc_to_choice` and the rest of config is the same as the task above, we can use the above configuration as a template by using `include: mmlu_high_school_geography.yaml` to load the config from that file. We'll need to add a unique task name as to not colide with the existing yaml config we're including. For this case we'll simply name this one `mmlu_high_school_geography_continuation`. `doc_to_text` is added here just for sake of clarity." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "lqElwU54TaK-" + }, + "outputs": [], + "source": [ + "YAML_mmlu_geo_string = \"\"\"\n", + "include: mmlu_high_school_geography.yaml\n", + "task: demo_mmlu_high_school_geography_continuation\n", + "doc_to_text: \"{{question.strip()}}\\nA. {{choices[0]}}\\nB. {{choices[1]}}\\nC. {{choices[2]}}\\nD. {{choices[3]}}\\nAnswer:\"\n", + "doc_to_choice: \"{{choices}}\"\n", + "\"\"\"\n", + "with open(\"mmlu_high_school_geography_continuation.yaml\", \"w\") as f:\n", + " f.write(YAML_mmlu_geo_string)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "id": "-_CVnDirdy7j" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-29:11:58:21,284 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", + "2023-11-29 11:58:22.850159: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2023-11-29 11:58:22.850219: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2023-11-29 11:58:22.850254: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2023-11-29 11:58:24.948103: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "2023-11-29:11:58:28,460 INFO [__main__.py:132] Verbosity set to INFO\n", + "2023-11-29:11:58:37,935 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", + "2023-11-29:11:58:37,935 INFO [__main__.py:143] Including path: ./\n", + "2023-11-29:11:58:37,969 INFO [__main__.py:205] Selected Tasks: ['demo_mmlu_high_school_geography_continuation']\n", + "2023-11-29:11:58:37,972 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", + "2023-11-29:11:58:38,008 INFO [huggingface.py:120] Using device 'cuda'\n", + "2023-11-29:11:58:59,758 INFO [task.py:355] Building contexts for task on rank 0...\n", + "2023-11-29:11:58:59,777 INFO [evaluator.py:319] Running loglikelihood requests\n", + "100% 40/40 [00:02<00:00, 16.23it/s]\n", + "fatal: not a git repository (or any of the parent directories): .git\n", + "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", + "| Tasks |Version|Filter|n-shot| Metric |Value| |Stderr|\n", + "|--------------------------------------------|-------|------|-----:|--------|----:|---|-----:|\n", + "|demo_mmlu_high_school_geography_continuation|Yaml |none | 0|acc | 0.1|ยฑ |0.1000|\n", + "| | |none | 0|acc_norm| 0.2|ยฑ |0.1333|\n", + "\n" + ] + } + ], + "source": [ + "# !accelerate launch --no_python\n", + "%env LOGLEVEL=DEBUG\n", + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", + " --include_path ./ \\\n", + " --tasks demo_mmlu_high_school_geography_continuation \\\n", + " --limit 10 \\\n", + " --output output/mmlu_high_school_geography_continuation/ \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-_CVnDirdy7j" + }, + "source": [ + "If we take a look at the samples, we can see that it is in fact evaluating the continuation based on the choices rather than the letters." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "id": "duBDqC6PAdjL" + }, + "outputs": [ + { + "data": { + "application/javascript": "\n ((filepath) => {{\n if (!google.colab.kernel.accessAllowed) {{\n return;\n }}\n google.colab.files.view(filepath);\n }})(\"/content/output/mmlu_high_school_geography_continuation/pretrained__EleutherAI__pythia-2.8b_demo_mmlu_high_school_geography_continuation.jsonl\")", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from google.colab import files\n", + "\n", + "\n", + "files.view(\n", + " \"output/mmlu_high_school_geography_continuation/pretrained__EleutherAI__pythia-2.8b_demo_mmlu_high_school_geography_continuation.jsonl\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6p0-KPwAgK5j" + }, + "source": [ + "## Closer Look at YAML Fields\n", + "\n", + "To prepare a task we can simply fill in a YAML config with the relevant information.\n", + "\n", + "`output_type`\n", + "The current provided evaluation types comprise of the following:\n", + "1. `loglikelihood`: Evaluates the loglikelihood of a continuation, conditioned on some input string.\n", + "2. `loglikelihood_rolling`: evaluate the loglikelihood of producing a string, conditioned on the empty string. (Used for perplexity evaluations)\n", + "3. `multiple_choice`: Evaluates loglikelihood among the a number of choices predicted by the model.\n", + "4. `greedy_until`: Model outputs greedy generation (can be configured to to use beam search and other generation-related parameters)\n", + "\n", + "The core prompt revolves around 3 fields.\n", + "1. `doc_to_text`: Denotes the prompt template that will be used as input to the model.\n", + "2. `doc_to_choice`: Available choices that will be used as continuation for the model. This is used when the `output_type` is `multiple_choice`, and otherwise can be left as `None`.\n", + "3. `doc_to_target`: When `output_type` is `multiple_choice`, this can be an index that corresponds to the correct answer, or the answer string itself (must be a subset of `doc_to_choice`). For other tasks, this is expected to be a string. You can fill this field with a feature name from the HF dataset so long as the resulting feature follows the conditioned described.\n", + "\n", + "These three fields can be expressed as strings, column names from the source dataset, or as Jinja2 templates that can use fields from the source dataset as variables.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6p0-KPwAgK5j" + }, + "source": [ + "## What if Jinja is not Sufficient?\n", + "\n", + "There can be times where the Jinja2 templating language is not enough to make the prompt we had in mind. There are a few ways to circumvent this limitation:\n", + "\n", + "1. Use `!function` operator for the prompt-related fields to pass a python function that takes as input the dataset row, and will output the prompt template component.\n", + "2. Perform a transformation on the dataset beforehand." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Below, we show an example of using `!function` to create `doc_to_text` from a python function:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "DYZ5c0JhR1lJ", + "outputId": "ca945235-fb9e-4f17-8bfa-78e7d6ec1490" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-29:11:59:08,312 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", + "2023-11-29 11:59:09.348327: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2023-11-29 11:59:09.348387: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2023-11-29 11:59:09.348421: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2023-11-29 11:59:10.573752: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "2023-11-29:11:59:14,044 INFO [__main__.py:132] Verbosity set to INFO\n", + "2023-11-29:11:59:23,654 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", + "2023-11-29:11:59:23,654 INFO [__main__.py:143] Including path: ./\n", + "2023-11-29:11:59:23,678 INFO [__main__.py:205] Selected Tasks: ['demo_mmlu_high_school_geography_function_prompt']\n", + "2023-11-29:11:59:23,679 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", + "2023-11-29:11:59:23,708 INFO [huggingface.py:120] Using device 'cuda'\n", + "2023-11-29:11:59:44,516 INFO [task.py:355] Building contexts for task on rank 0...\n", + "2023-11-29:11:59:44,524 INFO [evaluator.py:319] Running loglikelihood requests\n", + "100% 40/40 [00:02<00:00, 15.41it/s]\n", + "fatal: not a git repository (or any of the parent directories): .git\n", + "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", + "| Tasks |Version|Filter|n-shot| Metric |Value| |Stderr|\n", + "|-----------------------------------------------|-------|------|-----:|--------|----:|---|-----:|\n", + "|demo_mmlu_high_school_geography_function_prompt|Yaml |none | 0|acc | 0.1|ยฑ |0.1000|\n", + "| | |none | 0|acc_norm| 0.2|ยฑ |0.1333|\n", + "\n" + ] + } + ], + "source": [ + "YAML_mmlu_geo_string = \"\"\"\n", + "include: mmlu_high_school_geography.yaml\n", + "task: demo_mmlu_high_school_geography_function_prompt\n", + "doc_to_text: !function utils.doc_to_text\n", + "doc_to_choice: \"{{choices}}\"\n", + "\"\"\"\n", + "with open(\"demo_mmlu_high_school_geography_function_prompt.yaml\", \"w\") as f:\n", + " f.write(YAML_mmlu_geo_string)\n", + "\n", + "DOC_TO_TEXT = \"\"\"\n", + "def doc_to_text(x):\n", + " question = x[\"question\"].strip()\n", + " choices = x[\"choices\"]\n", + " option_a = choices[0]\n", + " option_b = choices[1]\n", + " option_c = choices[2]\n", + " option_d = choices[3]\n", + " return f\"{question}\\\\nA. {option_a}\\\\nB. {option_b}\\\\nC. {option_c}\\\\nD. {option_d}\\\\nAnswer:\"\n", + "\"\"\"\n", + "with open(\"utils.py\", \"w\") as f:\n", + " f.write(DOC_TO_TEXT)\n", + "\n", + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", + " --include_path ./ \\\n", + " --tasks demo_mmlu_high_school_geography_function_prompt \\\n", + " --limit 10 \\\n", + " --output output/demo_mmlu_high_school_geography_function_prompt/ \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we'll also show how to do this via preprocessing the dataset as necessary using the `process_docs` config field:\n", + "\n", + "We will write a function that will modify each document in our evaluation dataset's split to add a field that is suitable for us to use in `doc_to_text`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "YAML_mmlu_geo_string = \"\"\"\n", + "include: mmlu_high_school_geography.yaml\n", + "task: demo_mmlu_high_school_geography_function_prompt_2\n", + "process_docs: !function utils_process_docs.process_docs\n", + "doc_to_text: \"{{input}}\"\n", + "doc_to_choice: \"{{choices}}\"\n", + "\"\"\"\n", + "with open(\"demo_mmlu_high_school_geography_process_docs.yaml\", \"w\") as f:\n", + " f.write(YAML_mmlu_geo_string)\n", + "\n", + "DOC_TO_TEXT = \"\"\"\n", + "def process_docs(dataset):\n", + " def _process_doc(x):\n", + " question = x[\"question\"].strip()\n", + " choices = x[\"choices\"]\n", + " option_a = choices[0]\n", + " option_b = choices[1]\n", + " option_c = choices[2]\n", + " option_d = choices[3]\n", + " doc[\"input\"] = f\"{question}\\\\nA. {option_a}\\\\nB. {option_b}\\\\nC. {option_c}\\\\nD. {option_d}\\\\nAnswer:\"\n", + " return out_doc\n", + "\n", + " return dataset.map(_process_doc)\n", + "\"\"\"\n", + "\n", + "with open(\"utils_process_docs.py\", \"w\") as f:\n", + " f.write(DOC_TO_TEXT)\n", + "\n", + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", + " --include_path ./ \\\n", + " --tasks demo_mmlu_high_school_geography_function_prompt_2 \\\n", + " --limit 10 \\\n", + " --output output/demo_mmlu_high_school_geography_function_prompt_2/ \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We hope that this explainer gives you a sense of what can be done with and how to work with LM-Evaluation-Harnes v0.4.0 ! \n", + "\n", + "For more information, check out our documentation pages in the `docs/` folder, and if you have questions, please raise them in GitHub issues, or in #lm-thunderdome or #release-discussion on the EleutherAI discord server." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [ + "zAov81vTbL2K" + ], + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "46f521b73fd943c081c648fd873ebc0a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "48763b6233374554ae76035c0483066f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "4986a21eb560448fa79f4b25cde48951": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6b2d90209ec14230b3d58a74ac9b83bf": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7c5689bc13684db8a22681f41863dddd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a1d3a8aa016544a78e8821c8f6199e06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f61ed33fad754146bdd2ac9db1ba1c48", + "IPY_MODEL_bfa0af6aeff344c6845e1080a878e92e", + "IPY_MODEL_fd1ad9e0367d4004aae853b91c3a7617" + ], + "layout": "IPY_MODEL_6b2d90209ec14230b3d58a74ac9b83bf" + } + }, + "a73f357065d34d7baf0453ae4a8d75e2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "aed3acd2f2d74003b44079c333a0698e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bfa0af6aeff344c6845e1080a878e92e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7c5689bc13684db8a22681f41863dddd", + "max": 5669, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_48763b6233374554ae76035c0483066f", + "value": 5669 + } + }, + "f61ed33fad754146bdd2ac9db1ba1c48": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a73f357065d34d7baf0453ae4a8d75e2", + "placeholder": "โ€‹", + "style": "IPY_MODEL_46f521b73fd943c081c648fd873ebc0a", + "value": "Downloading builder script: 100%" + } + }, + "fd1ad9e0367d4004aae853b91c3a7617": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4986a21eb560448fa79f4b25cde48951", + "placeholder": "โ€‹", + "style": "IPY_MODEL_aed3acd2f2d74003b44079c333a0698e", + "value": " 5.67k/5.67k [00:00<00:00, 205kB/s]" + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/lm-evaluation-harness/examples/transformer-lens.py b/lm-evaluation-harness/examples/transformer-lens.py new file mode 100644 index 0000000000000000000000000000000000000000..e03576b121a5edd042a6d17fce3b541c78135521 --- /dev/null +++ b/lm-evaluation-harness/examples/transformer-lens.py @@ -0,0 +1,59 @@ +import warnings + +import torch +import torch.nn as nn +from transformer_lens import HookedTransformer +from transformers import AutoConfig + +from lm_eval import evaluator +from lm_eval.models.huggingface import HFLM + + +def evaluate_lm_eval(lens_model: HookedTransformer, tasks: list[str], **kwargs): + class HFLikeModelAdapter(nn.Module): + """Adapts HookedTransformer to match the HuggingFace interface expected by lm-eval""" + + def __init__(self, model: HookedTransformer): + super().__init__() + self.model = model + self.tokenizer = model.tokenizer + self.config = AutoConfig.from_pretrained(model.cfg.tokenizer_name) + self.device = model.cfg.device + self.tie_weights = lambda: self + + def forward(self, input_ids=None, attention_mask=None, **kwargs): + output = self.model(input_ids, attention_mask=attention_mask, **kwargs) + # Make sure output has the expected .logits attribute + if not hasattr(output, "logits"): + if isinstance(output, torch.Tensor): + output.logits = output + return output + + # Only delegate specific attributes we know we need + def to(self, *args, **kwargs): + return self.model.to(*args, **kwargs) + + def eval(self): + self.model.eval() + return self + + def train(self, mode=True): + self.model.train(mode) + return self + + model = HFLikeModelAdapter(lens_model) + warnings.filterwarnings("ignore", message="Failed to get model SHA for") + results = evaluator.simple_evaluate( + model=HFLM(pretrained=model, tokenizer=model.tokenizer), + tasks=tasks, + verbosity="WARNING", + **kwargs, + ) + return results + + +if __name__ == "__main__": + # Load base model + model = HookedTransformer.from_pretrained("pythia-70m") + res = evaluate_lm_eval(model, tasks=["arc_easy"]) + print(res["results"]) diff --git a/lm-evaluation-harness/examples/visualize-wandb.ipynb b/lm-evaluation-harness/examples/visualize-wandb.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..70d25fe608bba91199d12737b55054079d007c2d --- /dev/null +++ b/lm-evaluation-harness/examples/visualize-wandb.ipynb @@ -0,0 +1,172 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "fc477b96-adee-4829-a9d7-a5eb990df358", + "metadata": {}, + "source": [ + "# Visualizing Results in Weights and Biases\n", + "\n", + "With the Weights and Biases integration, you can now spend more time extracting deeper insights into your evaluation results. The integration is designed to streamline the process of logging and visualizing experiment results using the Weights & Biases (W&B) platform.\n", + "\n", + "The integration provide functionalities\n", + "\n", + "- to automatically log the evaluation results,\n", + "- log the samples as W&B Tables for easy visualization,\n", + "- log the `results.json` file as an artifact for version control,\n", + "- log the `_eval_samples.json` file if the samples are logged,\n", + "- generate a comprehensive report for analysis and visualization with all the important metric,\n", + "- log task and cli configs,\n", + "- and more out of the box like the command used to run the evaluation, GPU/CPU counts, timestamp, etc.\n", + "\n", + "The integration is super easy to use with the eval harness. Let's see how!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3851439a-bff4-41f2-bf21-1b3d8704913b", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# Install this project if you did not already have it.\n", + "# This is all that is needed to be installed to start using Weights and Biases\n", + "\n", + "!pip -qq install -e ..[wandb]" + ] + }, + { + "cell_type": "markdown", + "id": "8507fd7e-3b99-4a92-89fa-9eaada74ba91", + "metadata": {}, + "source": [ + "# Run the Eval Harness\n", + "\n", + "Run the eval harness as usual with a `wandb_args` flag. This flag is used to provide arguments for initializing a wandb run ([wandb.init](https://docs.wandb.ai/ref/python/init)) as comma separated string arguments.\n", + "\n", + "If `wandb_args` flag is used, the metrics and all other goodness will be automatically logged to Weights and Biases. In the stdout, you will find the link to the W&B run page as well as link to the generated report." + ] + }, + { + "cell_type": "markdown", + "id": "eec5866e-f01e-42f8-8803-9d77472ef991", + "metadata": {}, + "source": [ + "## Set your API Key\n", + "\n", + "Before you can use W&B, you need to authenticate your machine with an authentication key. Visit https://wandb.ai/authorize to get one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d824d163-71a9-4313-935d-f1d56397841c", + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "\n", + "\n", + "wandb.login()" + ] + }, + { + "cell_type": "markdown", + "id": "124e4a34-1547-4bed-bc09-db012bacbda6", + "metadata": {}, + "source": [ + "> Note that if you are using command line you can simply authenticate your machine by doing `wandb login` in your terminal. For more info check out the [documentation](https://docs.wandb.ai/quickstart#2-log-in-to-wb)." + ] + }, + { + "cell_type": "markdown", + "id": "abc6f6b6-179a-4aff-ada9-f380fb74df6e", + "metadata": {}, + "source": [ + "## Run and log to W&B" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd0a8130-a97b-451a-acd2-3f9885b88643", + "metadata": {}, + "outputs": [], + "source": [ + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=microsoft/phi-2,trust_remote_code=True \\\n", + " --tasks hellaswag,mmlu_abstract_algebra \\\n", + " --device cuda:0 \\\n", + " --batch_size 8 \\\n", + " --output_path output/phi-2 \\\n", + " --limit 10 \\\n", + " --wandb_args project=lm-eval-harness-integration \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "id": "e974cabdbe70b667", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "id": "5178ca9445b844e4", + "metadata": {}, + "source": [ + "W&B can also be initialized programmatically for use outside the CLI to parse and log the results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6a421b2cf3ddac5", + "metadata": {}, + "outputs": [], + "source": [ + "import lm_eval\n", + "from lm_eval.loggers import WandbLogger\n", + "\n", + "\n", + "results = lm_eval.simple_evaluate(\n", + " model=\"hf\",\n", + " model_args=\"pretrained=microsoft/phi-2,trust_remote_code=True\",\n", + " tasks=\"hellaswag,mmlu_abstract_algebra\",\n", + " log_samples=True,\n", + ")\n", + "\n", + "wandb_logger = WandbLogger(\n", + " project=\"lm-eval-harness-integration\", job_type=\"eval\"\n", + ") # or empty if wandb.init(...) already called before\n", + "wandb_logger.post_init(results)\n", + "wandb_logger.log_eval_result()\n", + "wandb_logger.log_eval_samples(results[\"samples\"]) # if log_samples" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/lm-evaluation-harness/examples/visualize-zeno.ipynb b/lm-evaluation-harness/examples/visualize-zeno.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..4ceabbf4253fde9e14d4cff5e088e1cbe66f457f --- /dev/null +++ b/lm-evaluation-harness/examples/visualize-zeno.ipynb @@ -0,0 +1,115 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Visualizing Results in Zeno\n", + "\n", + "Benchmarking your models is the first step towards making sure your model performs well.\n", + "However, looking at the data behind the benchmark, slicing the data into subsets, and comparing models on individual instances can help you even more in evaluating and quantifying the behavior of your AI system.\n", + "\n", + "All of this can be done in [Zeno](https://zenoml.com)!\n", + "Zeno is super easy to use with the eval harness, let's explore how you can easily upload and visualize your eval results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install this project if you did not already do that. This is all that needs to be installed for you to be able to visualize your data in Zeno!\n", + "!pip install -e ..\n", + "!pip install -e ..[zeno]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run the Eval Harness\n", + "\n", + "To visualize the results, run the eval harness with the `log_samples` and `output_path` flags. We expect `output_path` to contain multiple folders that represent individual model names. You can thus run your evaluation on any number of tasks and models and upload all of the results as projects on Zeno.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/gpt-neo-2.7B \\\n", + " --tasks hellaswag,wikitext \\\n", + " --batch_size 8 \\\n", + " --device mps \\\n", + " --log_samples \\\n", + " --output_path output/gpt-neo-2.7B \\\n", + " --limit 10" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Set your API Key\n", + "\n", + "This is so you can be authenticated with Zeno.\n", + "If you don't already have a Zeno account, first create an account on [Zeno Hub](https://hub.zenoml.com).\n", + "After logging in to Zeno Hub, generate your API key by clicking on your profile at the bottom left to navigate to your account page.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%env ZENO_API_KEY=YOUR_API_KEY" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Visualize Eval Results\n", + "\n", + "You can now use the `zeno_visualize` script to upload the results to Zeno.\n", + "\n", + "This will use all subfolders in `data_path` as different models and upload all tasks within these model folders to Zeno. If you run the eval harness on multiple tasks, the `project_name` will be used as a prefix and one project will be created per task.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!python ../scripts/zeno_visualize.py --data_path output --project_name \"Zeno Upload Test\"" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "zeno_projects", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/lm-evaluation-harness/ignore.txt b/lm-evaluation-harness/ignore.txt new file mode 100644 index 0000000000000000000000000000000000000000..de10b539b98c9e500d2d838ed3eb9bece95c00e2 --- /dev/null +++ b/lm-evaluation-harness/ignore.txt @@ -0,0 +1,8 @@ +ROUGE +rouge +nin +maka +mor +te +ond +extraversion diff --git a/lm-evaluation-harness/llama3-8b_eval.log b/lm-evaluation-harness/llama3-8b_eval.log new file mode 100644 index 0000000000000000000000000000000000000000..7297e9f78cec0f6bad85018c63b8c4ca27f26fa3 --- /dev/null +++ b/lm-evaluation-harness/llama3-8b_eval.log @@ -0,0 +1,428 @@ +nohup: ignoring input + +================================================== +ๅผ€ๅง‹่ฏ„ไผฐ๏ผšไปปๅŠก=triviaqa | ๅฐ‘ๆ ทๆœฌๆ•ฐ=0 | ๆจกๅž‹=Llama-3.1-8B +่พ“ๅ‡บ่ทฏๅพ„๏ผšresults2/Llama-3.1-8B/base_triviaqa.json +================================================== +2025-12-01:19:26:13 INFO [__main__:440] Selected Tasks: ['triviaqa'] +2025-12-01:19:26:13 INFO [evaluator:189] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234 +2025-12-01:19:26:13 INFO [evaluator:227] Initializing hf model, with arguments: {'pretrained': '/mnt/bn/life-mllm/users/cxr/quantization/models/meta-llama/Llama-3.1-8B'} +2025-12-01:19:26:14 WARNING [accelerate.utils.other:513] Detected kernel version 5.4.143, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. +2025-12-01:19:26:14 INFO [models.huggingface:137] Using device 'cuda:2' +2025-12-01:19:26:14 INFO [models.huggingface:382] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:2'} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading checkpoint shards: 0%| | 0/4 [00:00 Union[str, dict, None]: + if value is None: + return None + try: + return json.loads(value) + except json.JSONDecodeError: + if "{" in value: + raise argparse.ArgumentTypeError( + f"Invalid JSON: {value}. Hint: Use double quotes for JSON strings." + ) + return value + + +def _int_or_none_list_arg_type( + min_len: int, max_len: int, defaults: str, value: str, split_char: str = "," +): + def parse_value(item): + item = item.strip().lower() + if item == "none": + return None + try: + return int(item) + except ValueError: + raise argparse.ArgumentTypeError(f"{item} is not an integer or None") + + items = [parse_value(v) for v in value.split(split_char)] + num_items = len(items) + + if num_items == 1: + # Makes downstream handling the same for single and multiple values + items = items * max_len + elif num_items < min_len or num_items > max_len: + raise argparse.ArgumentTypeError( + f"Argument requires {max_len} integers or None, separated by '{split_char}'" + ) + elif num_items != max_len: + logging.warning( + f"Argument requires {max_len} integers or None, separated by '{split_char}'. " + "Missing values will be filled with defaults." + ) + default_items = [parse_value(v) for v in defaults.split(split_char)] + items.extend( + default_items[num_items:] + ) # extend items list with missing defaults + + return items + + +def check_argument_types(parser: argparse.ArgumentParser): + """ + Check to make sure all CLI args are typed, raises error if not + """ + for action in parser._actions: + if action.dest != "help" and not action.const: + if action.type is None: + raise ValueError( + f"Argument '{action.dest}' doesn't have a type specified." + ) + else: + continue + + +def setup_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--model", "-m", type=str, default="hf", help="Name of model e.g. `hf`" + ) + parser.add_argument( + "--tasks", + "-t", + default=None, + type=str, + metavar="task1,task2", + help="Comma-separated list of task names or task groupings to evaluate on.\nTo get full list of tasks, use one of the commands `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above", + ) + parser.add_argument( + "--model_args", + "-a", + default="", + type=try_parse_json, + help="""Comma separated string or JSON formatted arguments for model, e.g. `pretrained=EleutherAI/pythia-160m,dtype=float32` or '{"pretrained":"EleutherAI/pythia-160m","dtype":"float32"}'""", + ) + parser.add_argument( + "--num_fewshot", + "-f", + type=int, + default=None, + metavar="N", + help="Number of examples in few-shot context", + ) + parser.add_argument( + "--batch_size", + "-b", + type=str, + default=1, + metavar="auto|auto:N|N", + help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.", + ) + parser.add_argument( + "--max_batch_size", + type=int, + default=None, + metavar="N", + help="Maximal batch size to try with --batch_size auto.", + ) + parser.add_argument( + "--device", + type=str, + default=None, + help="Device to use (e.g. cuda, cuda:0, cpu).", + ) + parser.add_argument( + "--output_path", + "-o", + default=None, + type=str, + metavar="DIR|DIR/file.json", + help="Path where result metrics will be saved. Can be either a directory or a .json file. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.", + ) + parser.add_argument( + "--limit", + "-L", + type=float, + default=None, + metavar="N|0 argparse.Namespace: + check_argument_types(parser) + return parser.parse_args() + + +def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: + if not args: + # we allow for args to be passed externally, else we parse them ourselves + parser = setup_parser() + args = parse_eval_args(parser) + + if args.wandb_args: + wandb_args_dict = simple_parse_args_string(args.wandb_args) + wandb_config_args_dict = simple_parse_args_string(args.wandb_config_args) + wandb_logger = WandbLogger(wandb_args_dict, wandb_config_args_dict) + + utils.setup_logging(args.verbosity) + eval_logger = logging.getLogger(__name__) + os.environ["TOKENIZERS_PARALLELISM"] = "false" + + # update the evaluation tracker args with the output path and the HF token + if args.output_path: + args.hf_hub_log_args += f",output_path={args.output_path}" + if os.environ.get("HF_TOKEN", None): + args.hf_hub_log_args += f",token={os.environ.get('HF_TOKEN')}" + evaluation_tracker_args = simple_parse_args_string(args.hf_hub_log_args) + evaluation_tracker = EvaluationTracker(**evaluation_tracker_args) + + if args.predict_only: + args.log_samples = True + if (args.log_samples or args.predict_only) and not args.output_path: + raise ValueError( + "Specify --output_path if providing --log_samples or --predict_only" + ) + + if args.fewshot_as_multiturn and args.apply_chat_template is False: + raise ValueError( + "When `fewshot_as_multiturn` is selected, `apply_chat_template` must be set (either to `True` or to the chosen template name)." + ) + + if args.include_path is not None: + eval_logger.info(f"Including path: {args.include_path}") + metadata = ( + simple_parse_args_string(args.model_args) + if isinstance(args.model_args, str) + else args.model_args + if isinstance(args.model_args, dict) + else {} + ) | ( + args.metadata + if isinstance(args.metadata, dict) + else simple_parse_args_string(args.metadata) + ) + + task_manager = TaskManager(include_path=args.include_path, metadata=metadata) + + if "push_samples_to_hub" in evaluation_tracker_args and not args.log_samples: + eval_logger.warning( + "Pushing samples to the Hub requires --log_samples to be set. Samples will not be pushed to the Hub." + ) + + if args.limit: + eval_logger.warning( + " --limit SHOULD ONLY BE USED FOR TESTING." + "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT." + ) + if args.samples: + assert args.limit is None, ( + "If --samples is not None, then --limit must be None." + ) + if (samples := Path(args.samples)).is_file(): + args.samples = json.loads(samples.read_text()) + else: + args.samples = json.loads(args.samples) + + if args.tasks is None: + eval_logger.error("Need to specify task to evaluate.") + sys.exit() + elif args.tasks == "list": + print(task_manager.list_all_tasks()) + sys.exit() + elif args.tasks == "list_groups": + print(task_manager.list_all_tasks(list_subtasks=False, list_tags=False)) + sys.exit() + elif args.tasks == "list_tags": + print(task_manager.list_all_tasks(list_groups=False, list_subtasks=False)) + sys.exit() + elif args.tasks == "list_subtasks": + print(task_manager.list_all_tasks(list_groups=False, list_tags=False)) + sys.exit() + else: + if os.path.isdir(args.tasks): + import glob + + task_names = [] + yaml_path = os.path.join(args.tasks, "*.yaml") + for yaml_file in glob.glob(yaml_path): + config = utils.load_yaml_config(yaml_file) + task_names.append(config) + else: + task_list = args.tasks.split(",") + task_names = task_manager.match_tasks(task_list) + for task in [task for task in task_list if task not in task_names]: + if os.path.isfile(task): + config = utils.load_yaml_config(task) + task_names.append(config) + task_missing = [ + task for task in task_list if task not in task_names and "*" not in task + ] # we don't want errors if a wildcard ("*") task name was used + + if task_missing: + missing = ", ".join(task_missing) + eval_logger.error( + f"Tasks were not found: {missing}\n" + f"{utils.SPACING}Try `lm-eval --tasks list` for list of available tasks", + ) + raise ValueError( + f"Tasks not found: {missing}. Try `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above, or pass '--verbosity DEBUG' to troubleshoot task registration issues." + ) + + # Respect user's value passed in via CLI, otherwise default to True and add to comma-separated model args + if args.trust_remote_code: + eval_logger.info( + "Passed `--trust_remote_code`, setting environment variable `HF_DATASETS_TRUST_REMOTE_CODE=true`" + ) + # HACK: import datasets and override its HF_DATASETS_TRUST_REMOTE_CODE value internally, + # because it's already been determined based on the prior env var before launching our + # script--`datasets` gets imported by lm_eval internally before these lines can update the env. + import datasets + + datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True + + args.model_args = args.model_args + ",trust_remote_code=True" + ( + eval_logger.info(f"Selected Tasks: {task_names}") + if eval_logger.getEffectiveLevel() >= logging.INFO + else print(f"Selected Tasks: {task_names}") + ) + + request_caching_args = request_caching_arg_to_dict( + cache_requests=args.cache_requests + ) + + results = evaluator.simple_evaluate( + model=args.model, + model_args=args.model_args, + tasks=task_names, + num_fewshot=args.num_fewshot, + batch_size=args.batch_size, + max_batch_size=args.max_batch_size, + device=args.device, + use_cache=args.use_cache, + limit=args.limit, + samples=args.samples, + check_integrity=args.check_integrity, + write_out=args.write_out, + log_samples=args.log_samples, + evaluation_tracker=evaluation_tracker, + system_instruction=args.system_instruction, + apply_chat_template=args.apply_chat_template, + fewshot_as_multiturn=args.fewshot_as_multiturn, + gen_kwargs=args.gen_kwargs, + task_manager=task_manager, + predict_only=args.predict_only, + random_seed=args.seed[0], + numpy_random_seed=args.seed[1], + torch_random_seed=args.seed[2], + fewshot_random_seed=args.seed[3], + confirm_run_unsafe_code=args.confirm_run_unsafe_code, + metadata=metadata, + **request_caching_args, + ) + + if results is not None: + if args.log_samples: + samples = results.pop("samples") + dumped = json.dumps( + results, indent=2, default=handle_non_serializable, ensure_ascii=False + ) + if args.show_config: + print(dumped) + + batch_sizes = ",".join(map(str, results["config"]["batch_sizes"])) + + # Add W&B logging + if args.wandb_args: + try: + wandb_logger.post_init(results) + wandb_logger.log_eval_result() + if args.log_samples: + wandb_logger.log_eval_samples(samples) + except Exception as e: + eval_logger.info(f"Logging to Weights and Biases failed due to {e}") + + evaluation_tracker.save_results_aggregated( + results=results, samples=samples if args.log_samples else None + ) + + if args.log_samples: + for task_name, config in results["configs"].items(): + evaluation_tracker.save_results_samples( + task_name=task_name, samples=samples[task_name] + ) + + if ( + evaluation_tracker.push_results_to_hub + or evaluation_tracker.push_samples_to_hub + ): + evaluation_tracker.recreate_metadata_card() + + print( + f"{args.model} ({args.model_args}), gen_kwargs: ({args.gen_kwargs}), limit: {args.limit}, num_fewshot: {args.num_fewshot}, " + f"batch_size: {args.batch_size}{f' ({batch_sizes})' if batch_sizes else ''}" + ) + print(make_table(results)) + if "groups" in results: + print(make_table(results, "groups")) + + if args.wandb_args: + # Tear down wandb run once all the logging is done. + wandb_logger.run.finish() + + +if __name__ == "__main__": + cli_evaluate() diff --git a/lm-evaluation-harness/lm_eval/__pycache__/__init__.cpython-310.pyc b/lm-evaluation-harness/lm_eval/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..effbaf3cb6bf5abcabe9dde87c0ee708a1bbc86e Binary files /dev/null and b/lm-evaluation-harness/lm_eval/__pycache__/__init__.cpython-310.pyc differ diff --git a/lm-evaluation-harness/lm_eval/__pycache__/__init__.cpython-311.pyc b/lm-evaluation-harness/lm_eval/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d96c64c41e9d6a66903c5407c54bbdefeeffc24b Binary files /dev/null and b/lm-evaluation-harness/lm_eval/__pycache__/__init__.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/__pycache__/__main__.cpython-310.pyc b/lm-evaluation-harness/lm_eval/__pycache__/__main__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5b90f7c7a59993f6b412e2692f2ed389504b511 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/__pycache__/__main__.cpython-310.pyc differ diff --git a/lm-evaluation-harness/lm_eval/__pycache__/__main__.cpython-311.pyc b/lm-evaluation-harness/lm_eval/__pycache__/__main__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4491cbcae89cc24fa64e58c8a53d3a27cef1b8b3 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/__pycache__/__main__.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/__pycache__/evaluator.cpython-310.pyc b/lm-evaluation-harness/lm_eval/__pycache__/evaluator.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..941f843453cdcf70b42052c8f0541657298b4e05 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/__pycache__/evaluator.cpython-310.pyc differ diff --git a/lm-evaluation-harness/lm_eval/__pycache__/evaluator.cpython-311.pyc b/lm-evaluation-harness/lm_eval/__pycache__/evaluator.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6450d38b0f64c7bdc7bd5bff160df1c9eeb55e6 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/__pycache__/evaluator.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/__pycache__/evaluator_utils.cpython-310.pyc b/lm-evaluation-harness/lm_eval/__pycache__/evaluator_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c42a1d2edb05cae1c9263914f4b6d6c56e694fce Binary files /dev/null and b/lm-evaluation-harness/lm_eval/__pycache__/evaluator_utils.cpython-310.pyc differ diff --git a/lm-evaluation-harness/lm_eval/__pycache__/evaluator_utils.cpython-311.pyc b/lm-evaluation-harness/lm_eval/__pycache__/evaluator_utils.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e44a3acc366665c394ae5cbd6e21f4c849d1835 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/__pycache__/evaluator_utils.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/__pycache__/utils.cpython-310.pyc b/lm-evaluation-harness/lm_eval/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3a345b78c406336d0915ce0eca55b752b99e741 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/__pycache__/utils.cpython-310.pyc differ diff --git a/lm-evaluation-harness/lm_eval/__pycache__/utils.cpython-311.pyc b/lm-evaluation-harness/lm_eval/__pycache__/utils.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bfc79e2eff37ef6adae08a3421941a761543fd99 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/__pycache__/utils.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/api/__init__.py b/lm-evaluation-harness/lm_eval/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-evaluation-harness/lm_eval/api/__pycache__/__init__.cpython-310.pyc b/lm-evaluation-harness/lm_eval/api/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc1eaafc342a83aad9da104e6649a010c42eeab1 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/api/__pycache__/__init__.cpython-310.pyc differ diff --git a/lm-evaluation-harness/lm_eval/api/__pycache__/filter.cpython-310.pyc b/lm-evaluation-harness/lm_eval/api/__pycache__/filter.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..199d1055229066bb42488a2bb99bf377f6bf5244 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/api/__pycache__/filter.cpython-310.pyc differ diff --git a/lm-evaluation-harness/lm_eval/api/__pycache__/filter.cpython-311.pyc b/lm-evaluation-harness/lm_eval/api/__pycache__/filter.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f2236f2cfdbc521da463d4b0ca864eb36848fa62 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/api/__pycache__/filter.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/api/__pycache__/group.cpython-311.pyc b/lm-evaluation-harness/lm_eval/api/__pycache__/group.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a61ccb460a927bdc5e040530707edb963810c76 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/api/__pycache__/group.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/api/__pycache__/instance.cpython-311.pyc b/lm-evaluation-harness/lm_eval/api/__pycache__/instance.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..03db63022de1b01af446befe968fd50b15e0d3f8 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/api/__pycache__/instance.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/api/__pycache__/metrics.cpython-311.pyc b/lm-evaluation-harness/lm_eval/api/__pycache__/metrics.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..11182f1aab5dde910f15ce7b40435e105c098806 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/api/__pycache__/metrics.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/api/__pycache__/model.cpython-311.pyc b/lm-evaluation-harness/lm_eval/api/__pycache__/model.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f2821e6dc19a91078c8217b805a81aace73d3ee7 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/api/__pycache__/model.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/api/__pycache__/registry.cpython-310.pyc b/lm-evaluation-harness/lm_eval/api/__pycache__/registry.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..79427d26d988a22252919195aeb3b0a21e2781b1 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/api/__pycache__/registry.cpython-310.pyc differ diff --git a/lm-evaluation-harness/lm_eval/api/__pycache__/task.cpython-311.pyc b/lm-evaluation-harness/lm_eval/api/__pycache__/task.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b1564ed90f7c7d4b53e62282e548f79074de708 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/api/__pycache__/task.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/api/filter.py b/lm-evaluation-harness/lm_eval/api/filter.py new file mode 100644 index 0000000000000000000000000000000000000000..8d9db6821724c497c4a27116a1238e3b8d32ae29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/api/filter.py @@ -0,0 +1,56 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Callable, Iterable, List, Union + +from lm_eval.api.instance import Instance + + +class Filter(ABC): + """ + Filter classes operate on a per-task level. + They take all model outputs (`instance.resps` for all `task.instances`) + across all instances of a task, and perform operations. + In a single run, one can configure any number of separate filters or lists of filters. + + """ + + def __init__(self, **kwargs) -> None: + """ + Can define custom behavior here, if an individual instantiation of a Filter class should have state. + """ + + @abstractmethod + def apply(self, resps: Union[List, Iterable], docs: List[dict]) -> Iterable: + """ + Defines the operation to perform on a list of the `inst.resps` properties of `Instance` objects. + Should return the list of (filtered) response lists *in the same order as they were input*, e.g. + if pass in [, ] should return + [, ] + """ + return resps + + +@dataclass +class FilterEnsemble: + """ + FilterEnsemble creates a pipeline applying multiple filters. + Its intended usage is to stack multiple post-processing steps in order. + `task.apply_filters` should use a list of FilterEnsemble classes that it stores, to apply each + pipeline separately. + """ + + name: str + filters: List[Callable[[], Filter]] + + def apply(self, instances: List[Instance]) -> None: + resps, docs = zip(*((inst.resps, inst.doc) for inst in instances)) + resps, docs = list(resps), list(docs) + + for f in self.filters: + # apply filters in sequence + resps = f().apply(resps, docs) + + # add the end results after filtering to filtered_requests of their respective source instances. + # has key `self.name`: each FilterEnsemble applied in a given run should use a different name. + for inst, resp in zip(instances, resps): + inst.filtered_resps[self.name] = resp diff --git a/lm-evaluation-harness/lm_eval/api/group.py b/lm-evaluation-harness/lm_eval/api/group.py new file mode 100644 index 0000000000000000000000000000000000000000..0c60739bbd26c79ecab91f54240798b2ae9e3313 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/api/group.py @@ -0,0 +1,115 @@ +import abc +from dataclasses import asdict, dataclass +from inspect import getsource +from typing import Any, Callable, List, Optional, Union + + +@dataclass +class AggMetricConfig(dict): + metric: Optional[str] = None + aggregation: Optional[str] = "mean" + weight_by_size: Optional[str] = False + # list of filter names which should be incorporated into the aggregated metric. + filter_list: Optional[Union[str, list]] = "none" + + def __post_init__(self): + if self.aggregation != "mean" and not callable(self.aggregation): + raise ValueError( + f"Currently, 'mean' is the only pre-defined aggregation across groups' subtasks. Got '{self.aggregation}'." + ) + + if isinstance(self.filter_list, str): + self.filter_list = [self.filter_list] + + +@dataclass +class GroupConfig(dict): + group: Optional[str] = None + group_alias: Optional[str] = None + task: Optional[Union[str, list]] = None + aggregate_metric_list: Optional[ + Union[List[AggMetricConfig], AggMetricConfig, dict] + ] = None + metadata: Optional[dict] = ( + None # by default, not used in the code. allows for users to pass arbitrary info to tasks + ) + + def __getitem__(self, item): + return getattr(self, item) + + def __setitem__(self, item, value): + return setattr(self, item, value) + + def __post_init__(self): + if self.aggregate_metric_list is not None: + if isinstance(self.aggregate_metric_list, dict): + self.aggregate_metric_list = [self.aggregate_metric_list] + + self.aggregate_metric_list = [ + AggMetricConfig(**item) if isinstance(item, dict) else item + for item in self.aggregate_metric_list + ] + + def to_dict(self, keep_callable: bool = False) -> dict: + """dumps the current config as a dictionary object, as a printable format. + null fields will not be printed. + Used for dumping results alongside full task configuration + + :return: dict + A printable dictionary version of the TaskConfig object. + + # TODO: should any default value in the TaskConfig not be printed? + """ + cfg_dict = asdict(self) + # remove values that are `None` + for k, v in list(cfg_dict.items()): + if callable(v): + cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable) + return cfg_dict + + def serialize_function( + self, value: Union[Callable, str], keep_callable=False + ) -> Union[Callable, str]: + """Serializes a given function or string. + + If 'keep_callable' is True, the original callable is returned. + Otherwise, attempts to return the source code of the callable using 'getsource'. + """ + if keep_callable: + return value + else: + try: + return getsource(value) + except (TypeError, OSError): + return str(value) + + +class ConfigurableGroup(abc.ABC): + def __init__( + self, + config: Optional[dict] = None, + ) -> None: + self._config = GroupConfig(**config) + + @property + def group(self): + return self._config.group + + @property + def group_alias(self): + return self._config.group_alias + + @property + def version(self): + return self._config.version + + @property + def config(self): + return self._config.to_dict() + + @property + def group_name(self) -> Any: + return self._config.group + + def __repr__(self): + return f"ConfigurableGroup(group={self.group},group_alias={self.group_alias})" diff --git a/lm-evaluation-harness/lm_eval/api/instance.py b/lm-evaluation-harness/lm_eval/api/instance.py new file mode 100644 index 0000000000000000000000000000000000000000..d3c6afa0644e729ba441728c72a2469fdad07b8f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/api/instance.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass, field +from typing import Literal, Optional, Tuple + + +OutputType = Literal[ + "loglikelihood", "loglikelihood_rolling", "generate_until", "multiple_choice" +] + + +@dataclass +class Instance: + request_type: OutputType + doc: dict + arguments: tuple + idx: int + metadata: Tuple[Optional[str], Optional[int], Optional[int]] = field( + default_factory=lambda: (None, None, None) + ) + resps: list = field(default_factory=list) + filtered_resps: dict = field(default_factory=dict) + + # initialized after init + task_name: Optional[str] = None + doc_id: Optional[int] = None + repeats: Optional[int] = None + + def __post_init__(self) -> None: + # unpack metadata field + self.task_name, self.doc_id, self.repeats = self.metadata + + @property + def args(self): + """ + Returns (string,) where `string` is the string to calculate loglikelihood over + """ + return ( + self.arguments if isinstance(self.arguments, tuple) else (self.arguments,) + ) diff --git a/lm-evaluation-harness/lm_eval/api/metrics.py b/lm-evaluation-harness/lm_eval/api/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..61fca5e19d376502f3b75aa5328045cee6ee5454 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/api/metrics.py @@ -0,0 +1,578 @@ +import logging +import math +import random +import re +import string +from collections.abc import Iterable +from typing import List + +import numpy as np +import sacrebleu + +from lm_eval.api.registry import register_aggregation, register_metric + + +eval_logger = logging.getLogger(__name__) + + +# Register Aggregations First +@register_aggregation("bypass") +def bypass_agg(arr): + return 999 + + +@register_aggregation("nanmean") +def nanmean(arr): + if len(arr) == 0 or all(np.isnan(arr)): + return np.nan + return np.nanmean(arr) + + +@register_aggregation("mean") +def mean(arr): + return sum(arr) / len(arr) + + +@register_aggregation("median") +def median(arr): + return arr[len(arr) // 2] + + +# Certain metrics must be calculated across all documents in a benchmark. +# We use them as aggregation metrics, paired with no-op passthrough metric fns. +@register_aggregation("perplexity") +def perplexity(items): + return math.exp(-mean(items)) + + +@register_aggregation("weighted_perplexity") +def weighted_perplexity(items): + return math.exp(-weighted_mean(items)) + + +@register_aggregation("bits_per_byte") +def bits_per_byte(items): + return -weighted_mean(items) / math.log(2) + + +@register_aggregation("f1") +def f1_score(items): + from sklearn.metrics import f1_score + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + fscore = f1_score(golds, preds) + + return np.max(fscore) + + +@register_aggregation("matthews_corrcoef") +def matthews_corrcoef(items): + from sklearn.metrics import matthews_corrcoef + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + return matthews_corrcoef(golds, preds) + + +@register_aggregation("bleu") +def bleu(items): + """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric + for evaluating a generated sentence to a reference sentence. It counts matching + n-grams in the candidate translation to n-grams in the reference text, where + 1-gram or unigram would be each token and a bigram comparison would be each + word pair. The comparison is made regardless of word order + Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/ + Paper: https://www.aclweb.org/anthology/P02-1040/ + + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_bleu(preds, refs).score + + +@register_aggregation("chrf") +def chrf(items): + """chrF++ is a tool for automatic evaluation of machine translation output + based on character n-gram precision and recall enhanced with word n-grams. + Source: https://github.com/m-popovic/chrF + Paper: https://www.aclweb.org/anthology/W15-3049.pdf + + Higher is better # TODO I think + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_chrf(preds, refs).score + + +@register_aggregation("ter") +def ter(items): + """Translation Error Rate is an error metric for machine translation that + measures the number of edits required to change a system output into one + of the references + Source: http://www.cs.umd.edu/~snover/tercom/ + Paper: http://mt-archive.info/AMTA-2006-Snover.pdf + + Lower is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_ter(preds, refs).score + + +@register_aggregation("brier_score") +def brier_score(items): # This is a passthrough function + gold, predictions = list(zip(*items)) + bs, num_class = np.array(predictions).shape + + gold = list(gold) + gold_one_hot = np.eye(num_class)[gold] + return np.mean(np.sum((predictions - gold_one_hot) ** 2, axis=1)) + + +@register_metric( + metric="brier_score", + higher_is_better=False, + output_type=["multiple_choice"], + aggregation="brier_score", +) +def brier_score_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc", + higher_is_better=True, + output_type=["loglikelihood", "multiple_choice"], + aggregation="mean", +) +def acc_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc_norm", + higher_is_better=True, + output_type=["loglikelihood", "multiple_choice"], + aggregation="mean", +) +def acc_norm_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc_mutual_info", + higher_is_better=True, + output_type="multiple_choice", + aggregation="mean", +) +def acc_mutual_info_fn(items): # This is a passthrough function + return items + + +### the code used in the `exact_match_hf_evaluate` function is ported from +### https://github.com/huggingface/evaluate/blob/main/metrics/exact_match/exact_match.py +### which is under the apache license. + +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +def exact_match_hf_evaluate( + predictions, + references, + regexes_to_ignore=None, + ignore_case=False, + ignore_punctuation=False, + ignore_numbers=False, +): + if regexes_to_ignore is not None: + for s in regexes_to_ignore: + predictions = np.array([re.sub(s, "", x) for x in predictions]) + references = np.array([re.sub(s, "", x) for x in references]) + else: + predictions = np.asarray(predictions) + references = np.asarray(references) + + if ignore_case: + predictions = np.char.lower(predictions) + references = np.char.lower(references) + + if ignore_punctuation: + repl_table = string.punctuation.maketrans("", "", string.punctuation) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + if ignore_numbers: + repl_table = string.digits.maketrans("", "", string.digits) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + score_list = predictions == references + + return {"exact_match": np.mean(score_list)} + + +### + + +@register_metric( + metric="exact_match", + higher_is_better=True, + output_type="generate_until", + aggregation="mean", +) +def exact_match_fn(**kwargs): + return exact_match_hf_evaluate(**kwargs) + + +@register_metric( + metric="perplexity", + higher_is_better=False, + output_type="loglikelihood", + aggregation="perplexity", +) +def perplexity_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="word_perplexity", + higher_is_better=False, + output_type="loglikelihood_rolling", + aggregation="weighted_perplexity", +) +def word_perplexity_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="byte_perplexity", + higher_is_better=False, + output_type="loglikelihood_rolling", + aggregation="weighted_perplexity", +) +def byte_perplexity_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="bits_per_byte", + higher_is_better=False, + output_type="loglikelihood_rolling", + aggregation="bits_per_byte", +) +def bits_per_byte_fn(items): # This is a passthrough function + return items + + +def pop_stddev(arr): + mu = mean(arr) + return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr)) + + +def sample_stddev(arr): + mu = mean(arr) + return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1)) + + +def mean_stderr(arr): + return sample_stddev(arr) / math.sqrt(len(arr)) + + +@register_metric( + metric="bypass", + higher_is_better=True, + output_type=["loglikelihood", "multiple_choice", "generate_until"], + aggregation="bypass", +) +def bypass(items): + return None + + +@register_metric( + metric="mcc", + higher_is_better=True, + output_type="multiple_choice", + aggregation="matthews_corrcoef", +) +def mcc_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="f1", + higher_is_better=True, + output_type="multiple_choice", + aggregation="f1", +) +def f1_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="bleu", + higher_is_better=True, + output_type="generate_until", + aggregation="bleu", +) +def bleu_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="chrf", + higher_is_better=True, + output_type="generate_until", + aggregation="chrf", +) +def chrf_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="ter", + higher_is_better=True, + output_type="generate_until", + aggregation="ter", +) +def ter_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc_all", + higher_is_better=True, + output_type="loglikelihood", + aggregation="mean", +) +def acc_all(items): + # Only count as correct if all answers are labeled correctly for each question + question_scoring_dict = {} + preds = list(zip(*items))[0] + docs = list(zip(*items))[1] + + for doc, pred in zip(docs, preds): + paragraph_id = doc["idx"]["paragraph"] + question_id = doc["idx"]["question"] + if (paragraph_id, question_id) not in question_scoring_dict: + question_scoring_dict[(paragraph_id, question_id)] = [] + + gold_label = doc["label"] == 1 + + question_scoring_dict[(paragraph_id, question_id)].append(gold_label == pred) + acc = np.mean([int(all(x)) for x in question_scoring_dict.values()]) + return acc + + +def acc_all_stderr(items): + # Only count as correct if all answers are labeled correctly for each question + question_scoring_dict = {} + preds = list(zip(*items))[0] + docs = list(zip(*items))[1] + + for doc, pred in zip(docs, preds): + question_id = doc["idx"]["question"] + if question_id not in question_scoring_dict: + question_scoring_dict[question_id] = [] + + gold_label = doc["label"] == 1 + question_scoring_dict[question_id].append(gold_label == pred) + + acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()]) + return acc + + +def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): + """Compute max metric between prediction and each ground truth.""" + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def weighted_mean(items): + a, b = zip(*items) + return sum(a) / sum(b) + + +def is_non_str_iterable(obj): + return isinstance(obj, Iterable) and not isinstance(obj, str) + + +def _sacreformat(refs, preds): + """Format refs and preds for sacrebleu corpus calculation. It is very particular""" + # Sacrebleu expects (List[str], List[List[str]) + # e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...]) + + # Note [ref1_stream] is the first reference for each pred. + # So lists are size N and (M, N) for N preds and M possible refs for each pred + # This is a different order of dimensions that I would expect + + # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds + # Must become List[List[str]] with the inner list corresponding to preds + if not is_non_str_iterable(refs): + refs = list(refs) + if not is_non_str_iterable(refs[0]): + refs = [[ref] for ref in refs] + refs = list(zip(*refs)) + # Note the number of refs in each ref list much match the number of preds + + # We expect preds to be List[str] or List[List[str]]. Must become List[str] + if not is_non_str_iterable(preds): + preds = list(preds) + if is_non_str_iterable(preds[0]): + assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}" + preds = [pred[0] for pred in preds] + + return refs, preds + + +# stderr stuff + + +class _bootstrap_internal: + def __init__(self, f, n) -> None: + self.f = f + self.n = n + + def __call__(self, v): + i, xs = v + rnd = random.Random() + rnd.seed(i) + res = [] + for _ in range(self.n): + res.append(self.f(rnd.choices(xs, k=len(xs)))) + return res + + +def bootstrap_stderr(f, xs, iters): + import multiprocessing as mp + + pool = mp.Pool(mp.cpu_count()) + # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something + # equivalent to stderr calculated without Bessel's correction in the stddev. + # Unfortunately, I haven't been able to figure out what the right correction is + # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but + # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator) + # Thankfully, shouldn't matter because our samples are pretty big usually anyways + res = [] + chunk_size = min(1000, iters) + from tqdm import tqdm + + print("bootstrapping for stddev:", f.__name__) + for bootstrap in tqdm( + pool.imap( + _bootstrap_internal(f, chunk_size), + [(i, xs) for i in range(iters // chunk_size)], + ), + total=iters // chunk_size, + ): + # sample w replacement + res.extend(bootstrap) + + pool.close() + return sample_stddev(res) + + +def stderr_for_metric(metric, bootstrap_iters: int): + if bootstrap_iters <= 0: + # return no function (don't compute stderr) if bootstrap iters = 0 + return None + + bootstrappable = [ + median, + matthews_corrcoef, + f1_score, + perplexity, + bleu, + chrf, + ter, + nanmean, + ] + + if metric in bootstrappable: + return lambda x: bootstrap_stderr(metric, x, iters=bootstrap_iters) + + stderr = {mean: mean_stderr, acc_all: acc_all_stderr} + + return stderr.get(metric, None) + + +def pooled_sample_stderr(stderrs: List[float], sizes: List[int]): + # Used to aggregate bootstrapped stderrs across subtasks in a group, + # when we are weighting by the size of each subtask. + # + + assert len(stderrs) == len(sizes) + + # formula source: https://en.wikipedia.org/wiki/Pooled_variance + # and: https://stats.stackexchange.com/a/4841331 + # this empirically seems to match running `stderr_for_metric` on all instances + # from the subtasks concatenated with each other. + pooled_sample_var = ( + sum([(size - 1) * stderr**2 * size for size, stderr in zip(sizes, stderrs)]) + ) / (sum(sizes) - len(sizes)) + + return np.sqrt(pooled_sample_var / sum(sizes)) + + +def combined_sample_stderr(stderrs: List[float], sizes: List[int], metrics=None): + assert metrics is not None, ( + "Need to pass a list of each subtask's metric for this stderr aggregation" + ) + assert len(stderrs) == len(sizes) and len(sizes) == len(metrics) + + # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1390 for more documentation. + # This formula depends on sample means. + # removed because it seems to give erroneously huge stderrs for groupings of tasks + # and does not seem to match up with bootstrap-calculated stderrs for groups. + + ### don't use this unless a statistician has told you it's the right thing to do ### + + # accumulators: we'll aggregate pairwise N - 1 times + variance = stderrs[0] ** 2 + curr_size = sizes[0] + curr_score = metrics[0] + + for stderr, size, score in zip(stderrs[1:], sizes[1:], metrics[1:]): + curr_score = ((curr_score * curr_size) + (score * size)) / ( + curr_size + size + ) # NOTE: this assumes our aggregation fn is "mean" + + variance = ((curr_size - 1) * variance + (size - 1) * (stderr**2)) / ( + curr_size + size - 1 + ) + curr_size * size / ((curr_size + size) * (curr_size + size - 1)) * ( + curr_score - score + ) ** 2 + + return np.sqrt(variance) + + +def aggregate_subtask_metrics(metrics, sizes, weight_by_size=True): + # A helper function that is used to aggregate + # subtask scores cross-task. + # TODO: does not hold for non-mean aggregations + if not weight_by_size: + sizes = [1] * len(sizes) + + assert len(metrics) == len(sizes) + + return sum([metric * size for metric, size in zip(metrics, sizes)]) / sum(sizes) diff --git a/lm-evaluation-harness/lm_eval/api/registry.py b/lm-evaluation-harness/lm_eval/api/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..4673b157b1fc1eaed2eb40e7a1ad527ce1fcb595 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/api/registry.py @@ -0,0 +1,196 @@ +import logging +from typing import Callable, Dict, Union + +import evaluate as hf_evaluate + +from lm_eval.api.model import LM + + +eval_logger = logging.getLogger(__name__) + +MODEL_REGISTRY = {} + + +def register_model(*names): + # either pass a list or a single alias. + # function receives them as a tuple of strings + + def decorate(cls): + for name in names: + assert issubclass(cls, LM), ( + f"Model '{name}' ({cls.__name__}) must extend LM class" + ) + + assert name not in MODEL_REGISTRY, ( + f"Model named '{name}' conflicts with existing model! Please register with a non-conflicting alias instead." + ) + + MODEL_REGISTRY[name] = cls + return cls + + return decorate + + +def get_model(model_name): + try: + return MODEL_REGISTRY[model_name] + except KeyError: + raise ValueError( + f"Attempted to load model '{model_name}', but no model for this name found! Supported model names: {', '.join(MODEL_REGISTRY.keys())}" + ) + + +TASK_REGISTRY = {} +GROUP_REGISTRY = {} +ALL_TASKS = set() +func2task_index = {} + + +def register_task(name): + def decorate(fn): + assert name not in TASK_REGISTRY, ( + f"task named '{name}' conflicts with existing registered task!" + ) + + TASK_REGISTRY[name] = fn + ALL_TASKS.add(name) + func2task_index[fn.__name__] = name + return fn + + return decorate + + +def register_group(name): + def decorate(fn): + func_name = func2task_index[fn.__name__] + if name in GROUP_REGISTRY: + GROUP_REGISTRY[name].append(func_name) + else: + GROUP_REGISTRY[name] = [func_name] + ALL_TASKS.add(name) + return fn + + return decorate + + +OUTPUT_TYPE_REGISTRY = {} +METRIC_REGISTRY = {} +METRIC_AGGREGATION_REGISTRY = {} +AGGREGATION_REGISTRY: Dict[str, Callable[[], Dict[str, Callable]]] = {} +HIGHER_IS_BETTER_REGISTRY = {} +FILTER_REGISTRY = {} + +DEFAULT_METRIC_REGISTRY = { + "loglikelihood": [ + "perplexity", + "acc", + ], + "loglikelihood_rolling": ["word_perplexity", "byte_perplexity", "bits_per_byte"], + "multiple_choice": ["acc", "acc_norm"], + "generate_until": ["exact_match"], +} + + +def register_metric(**args): + # TODO: do we want to enforce a certain interface to registered metrics? + def decorate(fn): + assert "metric" in args + name = args["metric"] + + for key, registry in [ + ("metric", METRIC_REGISTRY), + ("higher_is_better", HIGHER_IS_BETTER_REGISTRY), + ("aggregation", METRIC_AGGREGATION_REGISTRY), + ]: + if key in args: + value = args[key] + assert value not in registry, ( + f"{key} named '{value}' conflicts with existing registered {key}!" + ) + + if key == "metric": + registry[name] = fn + elif key == "aggregation": + registry[name] = AGGREGATION_REGISTRY[value] + else: + registry[name] = value + + return fn + + return decorate + + +def get_metric(name: str, hf_evaluate_metric=False) -> Callable: + if not hf_evaluate_metric: + if name in METRIC_REGISTRY: + return METRIC_REGISTRY[name] + else: + eval_logger.warning( + f"Could not find registered metric '{name}' in lm-eval, searching in HF Evaluate library..." + ) + + try: + metric_object = hf_evaluate.load(name) + return metric_object.compute + except Exception: + eval_logger.error( + f"{name} not found in the evaluate library! Please check https://huggingface.co/evaluate-metric", + ) + + +def register_aggregation(name: str): + def decorate(fn): + assert name not in AGGREGATION_REGISTRY, ( + f"aggregation named '{name}' conflicts with existing registered aggregation!" + ) + + AGGREGATION_REGISTRY[name] = fn + return fn + + return decorate + + +def get_aggregation(name: str) -> Callable[[], Dict[str, Callable]]: + try: + return AGGREGATION_REGISTRY[name] + except KeyError: + eval_logger.warning(f"{name} not a registered aggregation metric!") + + +def get_metric_aggregation(name: str) -> Callable[[], Dict[str, Callable]]: + try: + return METRIC_AGGREGATION_REGISTRY[name] + except KeyError: + eval_logger.warning(f"{name} metric is not assigned a default aggregation!") + + +def is_higher_better(metric_name) -> bool: + try: + return HIGHER_IS_BETTER_REGISTRY[metric_name] + except KeyError: + eval_logger.warning( + f"higher_is_better not specified for metric '{metric_name}'!" + ) + + +def register_filter(name): + def decorate(cls): + if name in FILTER_REGISTRY: + eval_logger.info( + f"Registering filter `{name}` that is already in Registry {FILTER_REGISTRY}" + ) + FILTER_REGISTRY[name] = cls + return cls + + return decorate + + +def get_filter(filter_name: Union[str, Callable]) -> Callable: + try: + return FILTER_REGISTRY[filter_name] + except KeyError as e: + if callable(filter_name): + return filter_name + else: + eval_logger.warning(f"filter `{filter_name}` is not registered!") + raise e diff --git a/lm-evaluation-harness/lm_eval/api/samplers.py b/lm-evaluation-harness/lm_eval/api/samplers.py new file mode 100644 index 0000000000000000000000000000000000000000..5d1791bdb4f8ae06cf4168dcdfa4c6a5a9bbc823 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/api/samplers.py @@ -0,0 +1,232 @@ +import logging +import warnings +from functools import partial +from typing import TYPE_CHECKING, Iterable, Optional, Union + +import datasets + + +if TYPE_CHECKING: + from random import Random + + from lm_eval.api.task import ConfigurableTask, Task + +eval_logger = logging.getLogger("lm-eval") + + +class ContextSampler: + def __init__( + self, + docs: list[dict], + task: Union["Task", "ConfigurableTask"], + fewshot_indices: Optional[Iterable] = None, + rnd: Optional["Random"] = None, + ) -> None: + self.rnd = rnd + if not self.rnd: + raise ValueError( + "A `random.Random` generator argument must be provided to `rnd` of FewShotSampler!" + ) + + self.task = task + self.config = task._config + + self.target_delimiter = self.config.target_delimiter + self.fewshot_delimiter = self.config.fewshot_delimiter + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_text", None) is not None + ): + self.doc_to_text = partial( + self.task.doc_to_text, + doc_to_text=self.config.fewshot_config.get("doc_to_text", None), + ) + else: + self.doc_to_text = self.task.doc_to_text + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_target", None) is not None + ): + self.doc_to_target = partial( + self.task.doc_to_target, + doc_to_target=self.config.fewshot_config.get("doc_to_target", None), + ) + else: + self.doc_to_target = self.task.doc_to_target + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_choice", None) is not None + ): + self.doc_to_choice = partial( + self.task.doc_to_choice, + doc_to_choice=self.config.fewshot_config.get("doc_to_choice", None), + ) + else: + self.doc_to_choice = self.task.doc_to_choice + + self.docs = docs # HF dataset split, provided by task._fewshot_docs() + if fewshot_indices: # subset few-shot docs from + if not isinstance(self.docs, datasets.Dataset): + raise ValueError( + "Got `fewshot_indices` but fewshot_docs are not a HF dataset. Don't use both `fewshot_indices` and a user-defined few-shot sample list simultaneously" + ) + self.docs = self.docs.select(fewshot_indices) + + def get_context(self, doc: dict, num_fewshot: int, gen_prefix: str = None): + # draw an extra fewshot sample if using same split as evaluating on + prefix = gen_prefix + " " if gen_prefix else "" + n_samples = ( + num_fewshot + 1 + if self.config.fewshot_split == self.config.test_split + else num_fewshot + ) + + # draw `n_samples` docs from fewshot_docs + fewshotex = self.sample(n_samples) + + # get rid of the doc that's the one we're evaluating, if it's in the fewshot + # TODO: should we just stop people from using fewshot from same split as evaluating? + selected_docs = [x for x in fewshotex if x != doc][:num_fewshot] + + labeled_examples = "" + for doc in selected_docs: + doc_content = self.doc_to_text(doc) + doc_target = self.doc_to_target(doc) + if self.config.doc_to_choice is None or isinstance(doc_content, str): + labeled_examples += doc_content + else: + labeled_examples += self.doc_to_choice(doc)[doc_content] + + if doc_target != "": + if self.target_delimiter.isspace() and str(doc_target)[0].isspace(): + # TODO: add logger warn once here. + warnings.warn( + "Both target_delimiter and target start with a space. This may cause issues.", + Warning, + stacklevel=2, + ) + labeled_examples += self.target_delimiter + labeled_examples += prefix + labeled_examples += ( + str(doc_target[0]) + if isinstance(doc_target, list) + else doc_target + if self.config.doc_to_choice is None or isinstance(doc_target, str) + else str(self.doc_to_choice(doc)[doc_target]) + ) + labeled_examples += self.fewshot_delimiter + + return labeled_examples + + def get_chat_context( + self, + doc: dict, + num_fewshot: int, + fewshot_as_multiturn: bool = False, + gen_prefix: Optional[str] = None, + ): + # TODO: Do we need any other delimiter + prefix = gen_prefix + " " if gen_prefix else "" + chat_history = [] + # draw an extra fewshot sample if using same split as evaluating on + n_samples = ( + num_fewshot + 1 + if self.config.fewshot_split == self.config.test_split + else num_fewshot + ) + # draw `n_samples` docs from fewshot_docs + fewshotex = self.sample(n_samples) + + # get rid of the doc that's the one we're evaluating, if it's in the fewshot + # TODO: should we just stop people from using fewshot from same split as evaluating? + selected_docs = [x for x in fewshotex if x != doc][:num_fewshot] + + if fewshot_as_multiturn: + for doc in selected_docs: + doc_content = self.doc_to_text(doc) + doc_target = self.doc_to_target(doc) + chat_history.append( + { + "role": "user", + "content": doc_content + if self.config.doc_to_choice is None + or isinstance(doc_content, str) + else self.doc_to_choice(doc)[doc_content], + } + ) + chat_history.append( + { + "role": "assistant", + "content": prefix + str(doc_target[0]) + if isinstance(doc_target, list) + else prefix + doc_target + if self.config.doc_to_choice is None + or isinstance(doc_target, str) + else prefix + str(self.doc_to_choice(doc)[doc_target]), + } + ) + else: + # get fewshot context as one user turn + chat_history.append( + { + "role": "user", + "content": self.get_context( + doc, num_fewshot, gen_prefix=gen_prefix + ), + } + ) + + return chat_history + + def sample(self, n: int): + """ + Draw `n` samples from our fewshot docs. This method should be overridden by subclasses. + """ + + return self.rnd.sample(self.docs, n) + + +class FirstNSampler(ContextSampler): + def sample(self, n: int) -> None: + """ + Draw the first `n` samples in order from the specified split. + Used for tasks with "canonical" ordered fewshot examples, such as MMLU and CMMLU. + """ + assert n <= len(self.docs), ( + f"Error: number of fewshot samples requested exceeds the {len(self.docs)} that are available." + ) + return self.docs[:n] + + +class BalancedSampler(ContextSampler): + def sample(self, n: int) -> None: + """ + TODO: this should return approximately class-balanced samples from our fewshot examples. + TODO: what order should they be in? maybe random? + """ + + pass + + +class ManualSampler(ContextSampler): + def sample(self, n: int) -> None: + """ """ + pass + + +SAMPLER_REGISTRY = { + "default": ContextSampler, + "first_n": FirstNSampler, +} + + +def get_sampler(name: str): + try: + return SAMPLER_REGISTRY[name] + except KeyError: + raise ValueError( + f"Attempted to use contextsampler '{name}', but no sampling strategy for this name found! Supported model names: {', '.join(SAMPLER_REGISTRY.keys())}" + ) diff --git a/lm-evaluation-harness/lm_eval/api/task.py b/lm-evaluation-harness/lm_eval/api/task.py new file mode 100644 index 0000000000000000000000000000000000000000..ad334b48b0ed80575326a09f04298a296838842d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/api/task.py @@ -0,0 +1,1879 @@ +import abc +import ast +import logging +import random +import re +from collections.abc import Callable +from copy import deepcopy +from dataclasses import asdict, dataclass +from inspect import getsource +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, +) + +import datasets +import numpy as np +from tqdm import tqdm + +from lm_eval import utils +from lm_eval.api import samplers +from lm_eval.api.instance import Instance, OutputType +from lm_eval.api.metrics import bits_per_byte, mean, weighted_perplexity +from lm_eval.api.registry import ( + AGGREGATION_REGISTRY, + DEFAULT_METRIC_REGISTRY, + get_aggregation, + get_metric, + get_metric_aggregation, + is_higher_better, +) +from lm_eval.caching.cache import load_from_cache, save_to_cache +from lm_eval.filters import build_filter_ensemble +from lm_eval.prompts import get_prompt + + +ALL_OUTPUT_TYPES = [ + "loglikelihood", + "multiple_choice", + "loglikelihood_rolling", + "generate_until", +] + +eval_logger = logging.getLogger(__name__) + + +@dataclass +class TaskConfig(dict): + # task naming/registry + task: Optional[str] = None + task_alias: Optional[str] = None + tag: Optional[Union[str, list]] = None + # HF dataset options. + # which dataset to use, + # and what splits for what purpose + custom_dataset: Optional[Callable] = None + dataset_path: Optional[str] = None + dataset_name: Optional[str] = None + dataset_kwargs: Optional[dict] = None + training_split: Optional[str] = None + validation_split: Optional[str] = None + test_split: Optional[str] = None + fewshot_split: Optional[str] = ( + None # TODO: assert that this not None if num_fewshot > 0. (?) assert if this is same split as one evaluating (?) + ) + # formatting / prompting options. + # see docs/advanced_task_guide.md for more info + process_docs: Optional[Callable] = None + doc_to_text: Optional[Union[Callable, str]] = None + doc_to_target: Optional[Union[Callable, str]] = None + doc_to_image: Union[Callable, str] = None + doc_to_audio: Union[Callable, str] = None + unsafe_code: bool = False + doc_to_choice: Optional[Union[Callable, str, dict, list]] = None + process_results: Optional[Union[Callable, str]] = None + use_prompt: Optional[str] = None + description: str = "" + target_delimiter: str = " " + fewshot_delimiter: str = "\n\n" + fewshot_config: Optional[dict] = None + # runtime configuration options + num_fewshot: Optional[int] = None + # scoring options + metric_list: Optional[list] = None + output_type: OutputType = "generate_until" + generation_kwargs: Optional[dict] = None + repeats: int = 1 + filter_list: Optional[Union[str, list]] = None + should_decontaminate: bool = False + doc_to_decontamination_query: Optional[str] = None + gen_prefix: Optional[str] = None + metadata: Optional[dict] = ( + None # by default, not used in the code. allows for users to pass arbitrary info to tasks + ) + + def __post_init__(self) -> None: + if self.generation_kwargs is not None: + if self.output_type != "generate_until": + eval_logger.warning( + f"[{self.task}] passed `generation_kwargs`, but not using `output_type: generate_until`!" + ) + + if "temperature" in self.generation_kwargs: + self.generation_kwargs["temperature"] = float( + self.generation_kwargs["temperature"] + ) + + if "until" not in self.generation_kwargs: + eval_logger.warning( + f"{self.task}: No `until` specified in `generation_kwargs`! Defaulting to the fewshot_delimiter={repr(self.fewshot_delimiter)}" + ) + self.generation_kwargs["until"] = [self.fewshot_delimiter] + else: + if self.output_type == "generate_until": + # ensure that we greedily generate in absence of explicit arguments otherwise + self.generation_kwargs = { + "until": ( + None + if self.fewshot_delimiter is None + else [self.fewshot_delimiter] + ), + "do_sample": False, + "temperature": 0, + } + eval_logger.warning( + f"{self.task}: No `generation_kwargs` specified in task config, defaulting to {self.generation_kwargs}" + ) + + def __getitem__(self, item): + return getattr(self, item) + + def __setitem__(self, item, value): + return setattr(self, item, value) + + def to_dict(self, keep_callable: bool = False) -> dict: + """dumps the current config as a dictionary object, as a printable format. + null fields will not be printed. + Used for dumping results alongside full task configuration + + :return: dict + A printable dictionary version of the TaskConfig object. + + # TODO: should any default value in the TaskConfig not be printed? + """ + cfg_dict = asdict(self) + # remove values that are `None` + for k, v in list(cfg_dict.items()): + if v is None: + cfg_dict.pop(k) + elif k == "metric_list": + for metric_dict in v: + for metric_key, metric_value in metric_dict.items(): + if callable(metric_value): + metric_dict[metric_key] = self.serialize_function( + metric_value, keep_callable=keep_callable + ) + cfg_dict[k] = v + elif callable(v): + cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable) + return cfg_dict + + def serialize_function( + self, value: Union[Callable, str], keep_callable=False + ) -> Union[Callable, str]: + """Serializes a given function or string. + + If 'keep_callable' is True, the original callable is returned. + Otherwise, attempts to return the source code of the callable using 'getsource'. + """ + if keep_callable: + return value + else: + try: + return getsource(value) + except (TypeError, OSError): + return str(value) + + +class Task(abc.ABC): + """A task represents an entire benchmark including its dataset, problems, + answers, and evaluation methods. See BoolQ for a simple example implementation + + A `doc` can be any python object which represents one instance of evaluation. + This is usually a dictionary e.g. + {"question": ..., "answer": ...} or + {"question": ..., question, answer) + """ + + VERSION: Optional[Union[int, str]] = None + + # The name of the `Task` benchmark as denoted in the HuggingFace datasets Hub + # or a path to a custom `datasets` loading script. + DATASET_PATH: Optional[str] = None + + # The name of a subset within `DATASET_PATH`. + DATASET_NAME: Optional[str] = None + + OUTPUT_TYPE: Optional[OutputType] = None + + def __init__( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode: Optional[datasets.DownloadMode] = None, + config: Optional[Mapping] = None, # Union[dict, TaskConfig] + ) -> None: + """ + :param data_dir: str + Stores the path to a local folder containing the `Task`'s data files. + Use this to specify the path to manually downloaded data (usually when + the dataset is not publicly accessible). + :param cache_dir: str + The directory to read/write the `Task` dataset. This follows the + HuggingFace `datasets` API with the default cache directory located at: + `~/.cache/huggingface/datasets` + NOTE: You can change the cache location globally for a given process + to another directory: + `export HF_DATASETS_CACHE="/path/to/another/directory"` + :param download_mode: datasets.DownloadMode + How to treat pre-existing `Task` downloads and data. + - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS` + Reuse download and reuse dataset. + - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS` + Reuse download with fresh dataset. + - `datasets.DownloadMode.FORCE_REDOWNLOAD` + Fresh download and fresh dataset. + """ + self.download(data_dir, cache_dir, download_mode) + self._training_docs: Optional[list] = None + self._fewshot_docs: Optional[list] = None + self._instances: Optional[List[Instance]] = None + + self._config: TaskConfig = TaskConfig({**config}) if config else TaskConfig() + + self._filters = [build_filter_ensemble("none", [["take_first", None]])] + self.fewshot_rnd: Optional[random.Random] = ( + None # purposely induce errors in case of improper usage + ) + + def download( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode=None, + ) -> None: + """Downloads and returns the task dataset. + Override this method to download the dataset from a custom API. + + :param data_dir: str + Stores the path to a local folder containing the `Task`'s data files. + Use this to specify the path to manually downloaded data (usually when + the dataset is not publicly accessible). + :param cache_dir: str + The directory to read/write the `Task` dataset. This follows the + HuggingFace `datasets` API with the default cache directory located at: + `~/.cache/huggingface/datasets` + NOTE: You can change the cache location globally for a given process + by setting the shell environment variable, `HF_DATASETS_CACHE`, + to another directory: + `export HF_DATASETS_CACHE="/path/to/another/directory"` + :param download_mode: datasets.DownloadMode + How to treat pre-existing `Task` downloads and data. + - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS` + Reuse download and reuse dataset. + - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS` + Reuse download with fresh dataset. + - `datasets.DownloadMode.FORCE_REDOWNLOAD` + Fresh download and fresh dataset. + """ + self.dataset = datasets.load_dataset( + path=self.DATASET_PATH, + name=self.DATASET_NAME, + data_dir=data_dir, + cache_dir=cache_dir, + download_mode=download_mode, + ) + + @property + def config(self) -> TaskConfig: + """Returns the TaskConfig associated with this class.""" + return self._config + + @abc.abstractmethod + def has_training_docs(self): + """Whether the task has a training set""" + pass + + @abc.abstractmethod + def has_validation_docs(self): + """Whether the task has a validation set""" + pass + + @abc.abstractmethod + def has_test_docs(self): + """Whether the task has a test set""" + pass + + def training_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + return [] + + def validation_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + return [] + + def test_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + return [] + + def fewshot_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + if self.has_training_docs(): + return self.training_docs() + elif self.has_validation_docs(): + return self.validation_docs() + else: + if self.config.get("num_fewshot", 0) > 0: + eval_logger.warning( + f"[Task: {self.config.task}] has_training_docs and has_validation_docs are False" + ", using test_docs as fewshot_docs but this is not recommended." + ) + return self.test_docs() + + def _process_doc(self, doc: dict) -> dict: + """ + Override this to process (detokenize, strip, replace, etc.) individual + documents. This can be used in a map over documents of a data split. + E.g. `map(self._process_doc, self.dataset["validation"])` + + :return: dict + The processed version of the specified `doc`. + """ + return doc + + @property + def instances(self) -> List[Instance]: + """After calling `task.build_all_requests()`, tasks + maintain a list of the dataset instances which will be evaluated. + """ + return self._instances + + def fewshot_examples(self, k, rnd): + if self._training_docs is None: + self._training_docs = list(self.training_docs()) + + return rnd.sample(self._training_docs, k) + + def doc_to_decontamination_query(self, doc): + raise NotImplementedError( + "Override doc_to_decontamination_query with document specific decontamination query." + ) + + @abc.abstractmethod + def doc_to_text(self, doc): + pass + + @abc.abstractmethod + def doc_to_target(self, doc): + pass + + # not an abstractmethod because not every language-only task has to implement this + def doc_to_image(self, doc): + raise NotImplementedError + + def doc_to_audio(self, doc): + raise NotImplementedError + + def doc_to_prefix(self, doc): + return "" + + def build_all_requests( + self, + *, + limit: Union[int, None] = None, + samples: Optional[List[int]] = None, + rank: int = 0, + world_size: int = 1, + cache_requests: bool = False, + rewrite_requests_cache: bool = False, + system_instruction: Optional[str] = None, + apply_chat_template: bool = False, + fewshot_as_multiturn: bool = False, + chat_template: Optional[Callable] = None, + tokenizer_name: str = "", + ) -> None: + """Build a set of Instances for a task, and store them in task.instances""" + + # used with caching + og_limit = limit + + cache_key = f"requests-{self._config.task}-{self.config.num_fewshot}shot-rank{rank}-world_size{world_size}" + cache_key += "-chat_template" if apply_chat_template else "" + cache_key += "-fewshot_as_multiturn" if fewshot_as_multiturn else "" + cache_key += ( + f"-system_prompt_hash{utils.hash_string(system_instruction)}" + if system_instruction is not None + else "" + ) + cache_key += f"-tokenizer{tokenizer_name}" + + cached_instances = load_from_cache(file_name=cache_key, cache=cache_requests) + + if cache_requests and cached_instances and not rewrite_requests_cache: + cached_instances = cached_instances[:limit] + + flattened_instances = [ + instance + for instance_group in cached_instances + for instance in instance_group + ] + + self._instances = flattened_instances + return + + eval_logger.info(f"Building contexts for {self.config.task} on rank {rank}...") + + instances = [] + + # process all documents when caching is specified for simplicity + if ( + cache_requests + and (not cached_instances or rewrite_requests_cache) + and limit is not None + ): + limit = None + + doc_id_docs = list( + self.doc_iterator( + rank=rank, limit=limit, samples=samples, world_size=world_size + ) + ) + + num_docs = len(doc_id_docs) + + for doc_id, doc in tqdm( + doc_id_docs, + total=num_docs, + ): + # sample fewshot context #TODO: need to offset doc_id by rank now! + fewshot_ctx = self.fewshot_context( + doc, + 0 if self.config.num_fewshot is None else self.config.num_fewshot, + system_instruction, + apply_chat_template, + fewshot_as_multiturn, + chat_template, + gen_prefix=self.doc_to_prefix(doc), + ) + + # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute + inst = self.construct_requests( + doc=doc, + ctx=fewshot_ctx, + metadata=(self.config["task"], doc_id, self.config.repeats), + apply_chat_template=apply_chat_template, + chat_template=chat_template, + ) + + if not isinstance(inst, list): + inst = [inst] + + instances.append(inst) + + # now flatten, this is to allow slicing to work with pickles + + sliced_instances = instances[:og_limit] + + flattened_instances = [ + instance + for instance_group in sliced_instances + for instance in instance_group + ] + + self._instances = flattened_instances + + if len(self._instances) == 0: + raise ValueError("task.build_requests() did not find any docs!") + + if cache_requests and (not cached_instances or rewrite_requests_cache): + save_to_cache(file_name=cache_key, obj=instances) + + @abc.abstractmethod + def construct_requests(self, doc, ctx, **kwargs): + """Uses RequestFactory to construct Requests and returns an iterable of + Requests which will be sent to the LM. + + :param doc: + The document as returned from training_docs, validation_docs, or test_docs. + :param ctx: str + The context string, generated by fewshot_context. This includes the natural + language description, as well as the few shot examples, and the question + part of the document for `doc`. + :param doc_idx: int + The index of a document within `self.test_docs()` or `self.validation_docs()`, + whichever is the main split used. + :param repeats: int + TODO: update this docstring + The number of times each instance in a dataset is inferred on. Defaults to 1, + can be increased for techniques like majority voting. + """ + pass + + @abc.abstractmethod + def process_results(self, doc, results): + """Take a single document and the LM results and evaluates, returning a + dict where keys are the names of submetrics and values are the values of + the metric for that one document + + :param doc: + The document as returned from training_docs, validation_docs, or test_docs. + :param results: + The results of the requests created in construct_requests. + """ + pass + + @abc.abstractmethod + def aggregation(self): + """ + :returns: {str: [metric_score] -> float} + A dictionary where keys are the names of submetrics and values are + functions that aggregate a list of metric scores + """ + pass + + @abc.abstractmethod + def higher_is_better(self): + """ + :returns: {str: bool} + A dictionary where keys are the names of submetrics and values are + whether a higher value of the submetric is better + """ + pass + + def get_config(self, key: str) -> Any: + return getattr(self._config, key, None) + + @classmethod + def count_bytes(cls, doc): + """Used for byte-level perplexity metrics in rolling loglikelihood""" + return len(doc.encode("utf-8")) + + @classmethod + def count_words(cls, doc): + """Downstream loglikelihood_rolling perplexity tasks with custom word boundaries should override this!""" + return len(re.split(r"\s+", doc)) + + @utils.positional_deprecated + def fewshot_context(self, doc, num_fewshot, rnd=None, description=None, **kwargs): + """Returns a fewshot context string that is made up of a prepended description + (if provided), the `num_fewshot` number of examples, and an appended prompt example. + + :param doc: str + The document as returned from training_docs, validation_docs, or test_docs. + :param num_fewshot: int + The number of fewshot examples to provide in the returned context string. + :param rnd: random.Random + The pseudo-random number generator used to randomly sample examples. + WARNING: This is currently a required arg although it's optionalized with a default `None`. + :param description: str + The task's description that will be prepended to the fewshot examples. + :returns: str + The fewshot context. + """ + if rnd is None: + if self.fewshot_rnd is not None: + rnd = self.fewshot_rnd + else: + raise ValueError( + "A `random.Random` generator argument must be provided to `rnd`" + ) + + description = description if description else "" + + if num_fewshot == 0: + labeled_examples = "" + else: + # for sets with no training docs, draw from other set *but ensure no overlap with current doc* + if self.has_training_docs(): + fewshotex = self.fewshot_examples(k=num_fewshot, rnd=rnd) + else: + if self._fewshot_docs is None: + self._fewshot_docs = list( + self.validation_docs() + if self.has_validation_docs() + else self.test_docs() + ) + + fewshotex = rnd.sample(self._fewshot_docs, num_fewshot + 1) + + # get rid of the doc that's the one we're evaluating, if it's in the fewshot + fewshotex = [x for x in fewshotex if x != doc][:num_fewshot] + + labeled_examples = ( + "\n\n".join( + [ + self.doc_to_text(doc) + self.doc_to_target(doc) + for doc in fewshotex + ] + ) + + "\n\n" + ) + + example = self.doc_to_text(doc) + return description + labeled_examples + example + + def apply_filters(self) -> Optional[List[Instance]]: + """Iterates over FilterEnsembles and applies them to instances""" + if hasattr(self, "_filters"): + for f in self._filters: + f.apply(self._instances) + else: + eval_logger.warning("No filter defined, passing through instances") + return self._instances + + def dump_config(self) -> dict: + """Returns the config as a dictionary.""" + # TODO: this should only return the overrides applied to a non-YAML task's configuration. + # (num_fewshot) + return self.config.to_dict() + + def set_config(self, key: str, value: Any, update: bool = False) -> None: + """Set or update the configuration for a given key.""" + if key is None: + raise ValueError("Key must be provided.") + + if update: + current_value = getattr(self._config, key, {}) + if not isinstance(current_value, dict): + raise TypeError( + f"Expected a dict for key '{key}', got {type(current_value).__name__} instead." + ) + current_value.update(value) + else: + setattr(self._config, key, value) + + def override_metric(self, metric_name: str) -> None: + """ + Override the default metrics used for evaluation with custom metrics. + + Parameters: + - metric_name (str): The name of the custom metric to override. Should be registered in api.metrics. + """ + ( + self._metric_fn_list, + self._aggregation_list, + self._metric_fn_kwargs, + self._higher_is_better, + ) = ({}, {}, {}, {}) + self._metric_fn_list[metric_name] = get_metric(metric_name) + self._aggregation_list[metric_name] = get_metric_aggregation(metric_name) + self._higher_is_better[metric_name] = is_higher_better(metric_name) + self._metric_fn_kwargs[metric_name] = {} + if not isinstance(self, ConfigurableTask): + self.process_results = lambda x, y: {metric_name: get_metric(metric_name)} + self.aggregation = lambda: { + metric_name: get_metric_aggregation(metric_name) + } + setattr(self._config, "metric_list", [{"metric": metric_name}]) + setattr(self._config, "process_results", None) + + def set_fewshot_seed(self, seed: Optional[int] = None) -> None: + self.fewshot_rnd = random.Random(seed) + if hasattr(self, "sampler"): + self.sampler.rnd = self.fewshot_rnd + + @property + def eval_docs(self) -> Union[datasets.Dataset, List[dict]]: + if self.has_test_docs(): + return self.test_docs() + elif self.has_validation_docs(): + return self.validation_docs() + else: + raise ValueError( + f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!" + ) + + def doc_iterator( + self, + *, + rank: int = 0, + limit: Union[int, None] = None, + world_size: int = 1, + samples: Optional[List[int]] = None, + ) -> Iterator[Tuple[int, Any]]: + if samples: + n = len(self.eval_docs) + assert all([e < n for e in samples]), ( + f"Elements of --samples should be in the interval [0,k-1] where k is the number of total examples. In this case, k={n}." + ) + eval_logger.info( + f"{self.config.task}: Evaluating on {len(samples)} examples" + ) + doc_iterator = utils.create_iterator( + enumerate(x for i, x in enumerate(self.eval_docs) if i in samples), + rank=int(rank), + limit=None, # limit does not matter here since we are selecting samples directly + world_size=int(world_size), + ) + else: + limit = int(limit) if limit else None + doc_iterator = utils.create_iterator( + enumerate(self.eval_docs), + rank=int(rank), + limit=limit, + world_size=int(world_size), + ) + return doc_iterator + + +class ConfigurableTask(Task): + VERSION = "Yaml" + OUTPUT_TYPE = None + CONFIG = None + + def __init__( + self, + data_dir=None, + cache_dir=None, + download_mode=None, + config: Optional[dict] = None, + ) -> None: # TODO no super() call here + # Get pre-configured attributes + self._config = self.CONFIG + + # Use new configurations if there was no preconfiguration + if self.config is None: + self._config = TaskConfig(**config) + # Overwrite configs + else: + if config is not None: + self._config.__dict__.update(config) + + if self.config is None: + raise ValueError( + "Must pass a config to ConfigurableTask, either in cls.CONFIG or `config` kwarg" + ) + + if isinstance(self.config.metadata, dict): + if "version" in self.config.metadata: + self.VERSION = self.config.metadata["version"] + + if self.config.output_type is not None: + if self.config.output_type not in ALL_OUTPUT_TYPES: + raise ValueError( + f"Got invalid output_type '{self.config.output_type}', must be in '{','.join(ALL_OUTPUT_TYPES)}'" + ) + self.OUTPUT_TYPE = self.config.output_type + + if self.config.doc_to_image is not None: + # mark the task as requiring multimodality. + self.MULTIMODAL = True + + if self.config.doc_to_audio: + # mark the task as requiring multimodality. + self.MULTIMODAL = True + + if self.config.unsafe_code is not False: + self.UNSAFE_CODE = True + + if self.config.dataset_path is not None: + self.DATASET_PATH = self.config.dataset_path + + if self.config.dataset_name is not None: + self.DATASET_NAME = self.config.dataset_name + + self._metric_fn_list = {} + self._metric_fn_kwargs = {} + self._aggregation_list = {} + self._higher_is_better = {} + + if self.config.metric_list is None: + # TODO: handle this in TaskConfig.__post_init__ ? + _metric_list = DEFAULT_METRIC_REGISTRY[self.config.output_type] + + for metric_name in _metric_list: + self._metric_fn_list[metric_name] = get_metric(metric_name) + self._metric_fn_kwargs[metric_name] = {} + self._aggregation_list[metric_name] = get_metric_aggregation( + metric_name + ) + self._higher_is_better[metric_name] = is_higher_better(metric_name) + else: + for metric_config in self.config.metric_list: + if "metric" not in metric_config: + raise ValueError( + "'metric' key not provided for an entry in 'metric_list', must be specified!" + ) + metric_name = metric_config["metric"] + kwargs = { + key: metric_config[key] + for key in metric_config + if key + not in ["metric", "aggregation", "higher_is_better", "hf_evaluate"] + } + hf_evaluate_metric = ( + "hf_evaluate" in metric_config + and metric_config["hf_evaluate"] is True + ) + + if self.config.process_results is not None: + self._metric_fn_list[metric_name] = None + self._metric_fn_kwargs[metric_name] = {} + elif callable(metric_name): + metric_fn = metric_name.__call__ + metric_name = metric_name.__name__ + self._metric_fn_list[metric_name] = metric_fn + self._metric_fn_kwargs[metric_name] = kwargs + else: + self._metric_fn_list[metric_name] = get_metric( + metric_name, hf_evaluate_metric + ) + self._metric_fn_kwargs[metric_name] = kwargs + + if "aggregation" in metric_config: + agg_name = metric_config["aggregation"] + if isinstance(agg_name, str): + self._aggregation_list[metric_name] = get_aggregation(agg_name) + elif callable(agg_name): # noqa: E721 + self._aggregation_list[metric_name] = metric_config[ + "aggregation" + ] + else: + INV_AGG_REGISTRY = {v: k for k, v in AGGREGATION_REGISTRY.items()} + metric_agg = get_metric_aggregation(metric_name) + eval_logger.warning( + f"[Task: {self.config.task}] metric {metric_name} is defined, but aggregation is not. " + f"using default " + f"aggregation={INV_AGG_REGISTRY[metric_agg]}" + ) + self._aggregation_list[metric_name] = metric_agg + + if "higher_is_better" in metric_config: + self._higher_is_better[metric_name] = metric_config[ + "higher_is_better" + ] + else: + eval_logger.warning( + f"[Task: {self.config.task}] metric {metric_name} is defined, but higher_is_better is not. " + f"using default " + f"higher_is_better={is_higher_better(metric_name)}" + ) + self._higher_is_better[metric_name] = is_higher_better(metric_name) + + self.download(self.config.dataset_kwargs) + self._training_docs = None + self._fewshot_docs = None + + if self.config.filter_list is not None: + self._filters = [] + for filter_config in self.config.filter_list: + filter_name = filter_config["name"] + filter_functions = filter_config["filter"] + components = [] + for function in filter_functions: + kwargs = { + key: function[key] for key in function if key != "function" + } + components.append([function["function"], kwargs]) + filter_pipeline = build_filter_ensemble(filter_name, components) + self._filters.append(filter_pipeline) + else: + # TODO: handle repeats in a more general way rather than just discarding + eval_logger.debug( + "No custom filters defined. Using default 'take_first' filter for handling repeats." + ) + self._filters = [build_filter_ensemble("none", [["take_first", None]])] + + if self.config.use_prompt is not None: + eval_logger.info(f"loading prompt {self.config.use_prompt}") + self.prompt = get_prompt( + self.config.use_prompt, self.DATASET_PATH, self.DATASET_NAME + ) + else: + self.prompt = None + + if self.fewshot_docs() is not None: + self.fewshot_rnd = ( + random.Random() + ) # setting with no seed, to be overridden at a later time + config_sampler: Union[str, Callable] = ( + self.config.fewshot_config.get("sampler", "default") + if self.config.fewshot_config + else "default" + ) + if isinstance(config_sampler, str): + self.sampler = samplers.get_sampler(config_sampler)( + list(self.fewshot_docs()), self, rnd=self.fewshot_rnd + ) + elif callable(config_sampler) and issubclass( + config_sampler, samplers.ContextSampler + ): + self.sampler = config_sampler( + docs=list(self.fewshot_docs()), task=self, rnd=self.fewshot_rnd + ) + else: + raise TypeError( + f"fewshot_config.sampler should be a string or callable of ContextSampler type, " + f"not {type(config_sampler)}" + ) + + self.task_docs = self.eval_docs + + # Test One Doc + self.features = list(self.task_docs.features.keys()) + self.multiple_input = 0 + self.multiple_target = 0 + test_doc = self.task_docs[0] + test_text = self.doc_to_text(test_doc) + test_target = self.doc_to_target(test_doc) + + if self.config.doc_to_choice is not None: + test_choice = self.doc_to_choice(test_doc) + if not isinstance(test_choice, list): + eval_logger.error("doc_to_choice must return list") + else: + num_choice = len(test_choice) + + if isinstance(test_text, int): + eval_logger.debug( + "doc_to_text returned an int. Assuming multiple inputs." + ) + self.multiple_input = num_choice + else: + test_choice = None + + if isinstance(test_target, list): + eval_logger.debug( + "doc_to_target returned a list. Assuming multiple targets." + ) + self.multiple_target = len(test_target) + else: + if (isinstance(test_target, int)) and (test_choice is not None): + test_target = test_choice[test_target] + else: + test_target = str(test_target) + + if test_choice is not None: + check_choices = test_choice + else: + check_choices = [test_target] + if self.config.doc_to_choice is not None: + for choice in check_choices: + choice_has_whitespace = True if choice[0].isspace() else False + delimiter_has_whitespace = ( + True + if self.config.target_delimiter.rstrip() + != self.config.target_delimiter + else False + ) + + if delimiter_has_whitespace and choice_has_whitespace: + eval_logger.debug( + f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" have whitespace' + ) + elif (not delimiter_has_whitespace) and (not choice_has_whitespace): + eval_logger.debug( + f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" do not have whitespace, ignore if the language you are evaluating on does not require/use whitespace' + ) + + def download( + self, dataset_kwargs: Optional[Dict[str, Any]] = None, **kwargs + ) -> None: + if isinstance(self.config.custom_dataset, Callable): + eval_logger.warning( + f"{self.config.task}: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager." + + "\nFor example --metadata='{\"max_seq_lengths\":[4096, 8192]}'. For details see task Readme." + ) + self.dataset = self.config.custom_dataset( + **(self.config.metadata or {}), **(self.config.dataset_kwargs or {}) + ) + else: + self.dataset = datasets.load_dataset( + path=self.DATASET_PATH, + name=self.DATASET_NAME, + **dataset_kwargs if dataset_kwargs is not None else {}, + ) + + def has_training_docs(self) -> bool: + if self.config.training_split is not None: + return True + else: + return False + + def has_validation_docs(self) -> bool: + if self.config.validation_split is not None: + return True + else: + return False + + def has_test_docs(self) -> bool: + if self.config.test_split is not None: + return True + else: + return False + + def training_docs(self) -> datasets.Dataset: + if self.has_training_docs(): + if self.config.process_docs is not None: + return self.config.process_docs( + self.dataset[self.config.training_split] + ) + return self.dataset[self.config.training_split] + + def validation_docs(self) -> datasets.Dataset: + if self.has_validation_docs(): + if self.config.process_docs is not None: + return self.config.process_docs( + self.dataset[self.config.validation_split] + ) + return self.dataset[self.config.validation_split] + + def test_docs(self) -> datasets.Dataset: + if self.has_test_docs(): + if self.config.process_docs is not None: + return self.config.process_docs(self.dataset[self.config.test_split]) + return self.dataset[self.config.test_split] + + def fewshot_docs(self): + if self.config.fewshot_split is not None: + if self.config.process_docs is not None: + return self.config.process_docs(self.dataset[self.config.fewshot_split]) + return self.dataset[self.config.fewshot_split] + elif ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("samples", None) is not None + ): + if isinstance(self.config.fewshot_config["samples"], list): + return self.config.fewshot_config["samples"] + elif callable(self.config.fewshot_config["samples"]): + return self.config.fewshot_config["samples"]() + else: + raise Exception( + "`fewshot_config['samples']` was incorrectly defined in the configuration. It should be either a list of samples as a dict, or function returning this list." + ) + else: + if (self.config.num_fewshot is not None) and (self.config.num_fewshot > 0): + eval_logger.warning( + f"[Task: {self.config.task}] " + "num_fewshot > 0 but fewshot_split is None. " + "using preconfigured rule." + ) + return super().fewshot_docs() + + @staticmethod + def append_target_question( + labeled_examples: List[Dict[str, str]], + question: str, + fewshot_as_multiturn: bool = False, + gen_prefix: Optional[str] = None, + ) -> None: + """Adds a target question to the labeled examples list. + If fewshot_as_multiturn is True, or labeled_examples is empty, or the last entry is a system turn, appends the question as a new user entry. + Otherwise, it is appended to the last user entry, ensuring that the conversation alternates between the user and the assistant. + """ + if not fewshot_as_multiturn: + # if no messages or last message is system, append as new user entry + if len(labeled_examples) == 0 or labeled_examples[-1]["role"] == "system": + labeled_examples.append({"role": "user", "content": question}) + # if last message is user, append to it to avoid two user messages in a row + else: + labeled_examples[-1]["content"] += question + else: + # if fewshot_as_multiturn is True, append as next user entry (last is always assistant) + labeled_examples.append({"role": "user", "content": question}) + if gen_prefix: + labeled_examples.append({"role": "assistant", "content": gen_prefix}) + + @utils.positional_deprecated + def fewshot_context( + self, + doc: dict, + num_fewshot: int, + system_instruction: Optional[str] = None, + apply_chat_template: bool = False, + fewshot_as_multiturn: bool = False, + chat_template: Optional[Callable] = None, + gen_prefix: Optional[str] = None, + ) -> Union[str, List[str]]: + """Returns a fewshot context string that is made up of a prepended description + (if provided), the `num_fewshot` number of examples, and an appended prompt example. + + :param doc: str + The document as returned from training_docs, validation_docs, or test_docs. + :param num_fewshot: int + The number of fewshot examples to provide in the returned context string. + :param system_instruction: str + System instruction to be applied to the prompt. + :param apply_chat_template: bool + Whether to apply the chat template to the fewshot context. + :param fewshot_as_multiturn: bool + Whether to provide the fewshot examples as a multiturn conversation or a single user turn. + :param chat_template: + callable (from lm.apply_chat_template) that takes in a list[Dict] chat transcript and renders it into a string. + :param gen_prefix: + String to append after the <|assistant|> token. + :returns: str + The fewshot context. + """ + if apply_chat_template: + labeled_examples = [] + else: + labeled_examples = "" + + # get task description + if description := self.config.description: + description = utils.apply_template(self.config.description, doc) + + # create system prompt based on the provided system instruction and description + if system_instruction is not None and description: + system_prompt = ( + f"{system_instruction}{self.sampler.fewshot_delimiter}{description}" + ) + elif system_instruction is not None: + system_prompt = system_instruction + elif description: + system_prompt = description + else: + system_prompt = "" + + # add system prompt if specified + if system_prompt: + if apply_chat_template: + labeled_examples.append({"role": "system", "content": system_prompt}) + else: + labeled_examples = system_prompt + # if few-shot - append examples after the system prompt + if num_fewshot > 0: + if apply_chat_template: + labeled_examples.extend( + self.sampler.get_chat_context( + doc, + num_fewshot, + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + ) + else: + labeled_examples += self.sampler.get_context( + doc, num_fewshot, gen_prefix=gen_prefix + ) + + example = self.doc_to_text(doc) + if apply_chat_template: + if self.multiple_input: + # TODO: append prefill? + if not labeled_examples: + return "" + return chat_template(labeled_examples) + if isinstance(example, str): + self.append_target_question( + labeled_examples, + example, + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + # for loglikelihood create a list of questions with appended choices + elif isinstance(example, list): + labeled_examples_list = [] + # copy chat history for each example and append the answer + for ex in example: + chat = deepcopy(labeled_examples) + self.append_target_question( + chat, + ex, + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + # TODO: append prefill? + labeled_examples_list.append( + chat_template( + chat, + add_generation_prompt=False if gen_prefix else True, + ) + ) + return labeled_examples_list + # if example is an integer, append the choice or convert to string + elif isinstance(example, int): + if self.config.doc_to_choice is not None: + choices = self.doc_to_choice(doc) + self.append_target_question( + labeled_examples, + choices[example], + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + else: + self.append_target_question( + labeled_examples, + str(example), + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + # return lm.apply_chat_template(labeled_examples) + return chat_template( + labeled_examples, + add_generation_prompt=False if gen_prefix else True, + ) + else: + prefix = ( + self.config.target_delimiter + gen_prefix + if gen_prefix is not None + else "" + ) + if self.multiple_input: + return labeled_examples + if isinstance(example, str): + return labeled_examples + example + prefix + elif isinstance(example, list): + return [labeled_examples + ex + prefix for ex in example] + elif isinstance(example, int): + if self.config.doc_to_choice is not None: + choices = self.doc_to_choice(doc) + return labeled_examples + choices[example] + prefix + else: + return labeled_examples + str(example) + prefix + + def apply_filters(self) -> Optional[List[Instance]]: + """Iterates over FilterEnsembles and applies them to instances""" + if hasattr(self, "_filters"): + for f in self._filters: + f.apply(self._instances) + else: + eval_logger.warning("No filter defined, passing through instances") + return self._instances + + def should_decontaminate(self): + return self.config.should_decontaminate + + def doc_to_decontamination_query(self, doc: dict): + if self.config.should_decontaminate: + if self.config.doc_to_decontamination_query is None: + return self.doc_to_text(doc) + else: + doc_to_decontamination_query = self.config.doc_to_decontamination_query + if doc_to_decontamination_query in self.features: + return doc[doc_to_decontamination_query] + elif callable(doc_to_decontamination_query): + return doc_to_decontamination_query(doc) + else: + return ast.literal_eval( + utils.apply_template( + self.config.doc_to_decontamination_query, doc + ) + ) + + def _process_doc(self, doc: dict) -> dict: + """ + Override this to process (detokenize, strip, replace, etc.) individual + documents. This can be used in a map over documents of a data split. + E.g. `map(self._process_doc, self.dataset["validation"])` + + :return: dict + The processed version of the specified `doc`. + """ + return doc + + def doc_to_text(self, doc, doc_to_text=None): + if self.prompt is not None: + doc_to_text = self.prompt + elif doc_to_text is not None: + doc_to_text = doc_to_text + else: + doc_to_text = self.config.doc_to_text + + if isinstance(doc_to_text, int): + return doc_to_text + elif isinstance(doc_to_text, str): + if doc_to_text in self.features: + # if self.config.doc_to_choice is not None: + # return self.doc_to_choice(doc)[doc[doc_to_text]] + # else: + return doc[doc_to_text] + else: + text_string = utils.apply_template(doc_to_text, doc) + if text_string.isdigit() and self._config.doc_to_choice is not None: + return ast.literal_eval(text_string) + else: + return text_string + elif callable(doc_to_text): + return doc_to_text(doc) + # Used when applying a Promptsource template + elif hasattr(doc_to_text, "apply"): + applied_prompt = doc_to_text.apply(doc) + if len(applied_prompt) == 2: + return applied_prompt[0] + else: + eval_logger.warning("Applied prompt returns empty string") + return self.config.fewshot_delimiter + else: + print(type(doc_to_text)) + raise TypeError + + def doc_to_target(self, doc: Mapping, doc_to_target=None) -> Union[int, str, list]: + if self.prompt is not None: + doc_to_target = self.prompt + elif doc_to_target is not None: + doc_to_target = doc_to_target + else: + doc_to_target = self.config.doc_to_target + + if isinstance(doc_to_target, int): + return doc_to_target + elif isinstance(doc_to_target, str): + if doc_to_target in self.features: + # if self.config.doc_to_choice is not None: + # return self.doc_to_choice(doc)[doc[doc_to_target]] + # else: + return doc[doc_to_target] + else: + target_string = utils.apply_template(doc_to_target, doc) + if target_string.isdigit() and self._config.doc_to_choice is not None: + return ast.literal_eval(target_string) + elif ( + len(target_string) >= 2 + and (target_string[0] == "[") + and (target_string[-1] == "]") + ): + try: + return ast.literal_eval(target_string) + except (SyntaxError, ValueError): + return target_string + else: + return target_string + elif isinstance(doc_to_target, list): + return doc_to_target + elif callable(doc_to_target): + return doc_to_target(doc) + # Used when applying a Promptsource template + elif hasattr(doc_to_target, "apply"): + applied_prompt = doc_to_target.apply(doc) + if len(applied_prompt) == 2: + return applied_prompt[1] + else: + eval_logger.warning("Applied prompt returns empty string") + return self.config.fewshot_delimiter + else: + raise TypeError + + def doc_to_choice(self, doc: Any, doc_to_choice=None) -> List[str]: + if self.prompt is not None: + doc_to_choice = self.prompt + elif doc_to_choice is not None: + doc_to_choice = doc_to_choice + elif self.config.doc_to_choice is None: + eval_logger.error("doc_to_choice was called but not set in config") + else: + doc_to_choice = self.config.doc_to_choice + + if isinstance(doc_to_choice, str): + if doc_to_choice in self.features: + return doc[doc_to_choice] + else: + return ast.literal_eval(utils.apply_template(doc_to_choice, doc)) + elif isinstance(doc_to_choice, list): + return doc_to_choice + elif isinstance(doc_to_choice, dict): + return list(doc_to_choice.values()) + elif callable(doc_to_choice): + return doc_to_choice(doc) + elif hasattr(doc_to_choice, "get_answer_choices_list"): + return doc_to_choice.get_answer_choices_list(doc) + else: + raise TypeError + + def doc_to_image(self, doc: Any, doc_to_image=None) -> Union[int, str, list]: + if doc_to_image is not None: + doc_to_image = doc_to_image + elif self.config.doc_to_image is not None: + doc_to_image = self.config.doc_to_image + else: + return None + + if isinstance(doc_to_image, list): + image_feature = [ + self.doc_to_image(doc, feature) for feature in doc_to_image + ] + return [feature for feature in image_feature if feature is not None] + elif isinstance(doc_to_image, str): + if doc_to_image in self.features: + return doc[doc_to_image] + else: + return ast.literal_eval(utils.apply_template(doc_to_image, doc)) + elif callable(doc_to_image): + return doc_to_image(doc) + else: + return None + + def doc_to_audio(self, doc: Any, doc_to_audio=None) -> Union[int, str, list]: + if doc_to_audio is not None: + doc_to_audio = doc_to_audio + elif self.config.doc_to_audio is not None: + doc_to_audio = self.config.doc_to_audio + else: + return None + + if isinstance(doc_to_audio, list): + audio_feature = [ + self.doc_to_audio(doc, feature) for feature in doc_to_audio + ] + return [feature for feature in audio_feature if feature is not None] + elif isinstance(doc_to_audio, str): + if doc_to_audio in self.features: + return doc[doc_to_audio] + else: + return ast.literal_eval(utils.apply_template(doc_to_audio, doc)) + elif callable(doc_to_audio): + return doc_to_audio(doc) + else: + return None + + def doc_to_prefix(self, doc): + if (gen_prefix := self.config.gen_prefix) is not None: + if gen_prefix in self.features: + return doc[gen_prefix] + else: + return utils.apply_template(gen_prefix, doc) + return None + + def construct_requests( + self, doc: dict, ctx: str, **kwargs + ) -> Union[List[Instance], Instance]: + apply_chat_template = kwargs.pop("apply_chat_template", False) + chat_template: Callable | None = kwargs.pop("chat_template", None) + + aux_arguments = None + + if self.OUTPUT_TYPE == "loglikelihood": + arguments = (ctx, self.doc_to_target(doc)) + elif self.OUTPUT_TYPE == "loglikelihood_rolling": + arguments = (self.doc_to_target(doc),) + elif self.OUTPUT_TYPE == "multiple_choice": + choices = self.doc_to_choice(doc) + target_delimiter = self.config.target_delimiter + if apply_chat_template: + target_delimiter = "" + if self.multiple_input: + # If there are multiple inputs, choices are placed in the ctx + # apply chat_template to choices if apply_chat_template + cont = self.doc_to_target(doc) + + arguments = [ + ( + ctx + + ( + chat_template([{"role": "user", "content": choice}]) + if apply_chat_template + else choice + ), + f"{target_delimiter}{cont}", + ) + for choice in choices + ] + else: + # Otherwise they are placed in the continuation + arguments = [(ctx, f"{target_delimiter}{cont}") for cont in choices] + + # TODO: we should raise a warning telling users this will at most ~2x runtime. + if "acc_mutual_info" in self._metric_fn_list.keys(): + # if we are calculating multiple choice accuracy + # using mutual information instead of raw loglikelihood as metric, need unconditional lls. + + # here mutual info refers to calculating + # log(P(choice|ctx) / P(choice)) = log(P(choice|ctx)) - log(P(choice)) + # in other words normalizing by subtracting the unconditional logprob of each choice. + # TODO: should these be strided? will have to modify the processing in process_results if so + aux_arguments = [ + ("", f"{target_delimiter}{choice}") for choice in choices + ] + + arguments.extend(aux_arguments) + + elif self.OUTPUT_TYPE == "generate_until": + arguments = (ctx, deepcopy(self.config.generation_kwargs)) + + multimodal_arg = {} + if ( + self.config.doc_to_image + ): # TODO: ensure that non-multimodal tasks aren't getting visual args + multimodal_arg = { + **multimodal_arg, + **{"visual": self.doc_to_image(doc)}, + } + + if ( + self.config.doc_to_audio + ): # TODO: ensure that non-multimodal tasks aren't getting audio args + multimodal_arg = { + **multimodal_arg, + **{"audio": self.doc_to_audio(doc)}, + } + + if bool(multimodal_arg): + if isinstance(arguments, list): + arguments = [arg + (multimodal_arg,) for arg in arguments] + else: + arguments = arguments + (multimodal_arg,) + + if self.OUTPUT_TYPE == "multiple_choice": + request_list = [ + Instance( + request_type="loglikelihood", + doc=doc, + arguments=arg, + idx=i, + **kwargs, + ) + for i, arg in enumerate(arguments) + ] + + return request_list + + return Instance( + request_type=self.OUTPUT_TYPE, + doc=doc, + arguments=arguments, + idx=0, + **kwargs, + ) + + def process_results(self, doc, results): + if callable(self.config.process_results): + return self.config.process_results(doc, results) + + result_dict = {} + use_metric = list(self._metric_fn_list.keys()) + if self.OUTPUT_TYPE == "loglikelihood": + results = results[0] + ll, is_greedy = results + return { + **({"perplexity": ll} if "perplexity" in use_metric else {}), + **({"acc": int(is_greedy)} if "acc" in use_metric else {}), + } + elif self.OUTPUT_TYPE == "loglikelihood_rolling": + (loglikelihood,) = results + _words = self.count_words(self.doc_to_target(doc)) + _bytes = self.count_bytes(self.doc_to_target(doc)) + return { + **( + {"word_perplexity": (loglikelihood, _words)} + if "word_perplexity" in use_metric + else {} + ), + **( + {"byte_perplexity": (loglikelihood, _bytes)} + if "byte_perplexity" in use_metric + else {} + ), + **( + {"bits_per_byte": (loglikelihood, _bytes)} + if "bits_per_byte" in use_metric + else {} + ), + } + elif self.OUTPUT_TYPE == "multiple_choice": + lls, is_greedy = zip(*results) + + # retrieve choices in List[str] form, to compute choice lengths, etc. + choices = self.doc_to_choice(doc) + completion_len = np.array([float(len(i)) for i in choices]) + + if ( + 2 * len(choices) == len(lls) + and "acc_mutual_info" in self._metric_fn_list.keys() + ): + # then we are doing mutual info. + # this stores the "dryrun" / unconditional answer loglikelihoods + # as we extend the args list with unconditional ("", continuation) pairs + lls_unconditional = lls[len(choices) :] + if len(lls_unconditional) != len(choices): + raise ValueError + # and this stores our "regular" conditional loglikelihoods + lls = lls[: len(choices)] + + pred = np.argmax(lls) + pred_norm = np.argmax(lls / completion_len) + + if self.multiple_input: + gold = self.doc_to_text(doc) + else: + gold = self.doc_to_target(doc) + + gold_index_error = False + if isinstance(gold, list): + gold = [i if i < len(choices) else -100 for i in gold] + if -100 in gold: + gold_index_error = True + else: + if isinstance(gold, int): + gold = gold if gold < len(choices) else -100 + elif isinstance(gold, str): + gold = choices.index(gold) if gold in choices else -100 + + if gold == -100: + gold_index_error = True + + if gold_index_error: + eval_logger.warning( + f"Label index was not in within range of available choices," + f"Sample:\n\n{doc}\n\n" + ) + + if self.multiple_target: + acc = 1.0 if pred in gold else 0.0 + acc_norm = 1.0 if pred_norm in gold else 0.0 + exact_match = int(any([is_greedy[i] if i != -100 else 0 for i in gold])) + else: + acc = 1.0 if pred == gold else 0.0 + acc_norm = 1.0 if pred_norm == gold else 0.0 + # TODO: this gets score of 0 on arc_challenge for pythia-70m. need to test that this works properly + exact_match = int(is_greedy[gold]) if gold != -100 else 0 + + prob_norm = utils.softmax(lls) + + # TODO use keyword arguments to the metric? + # gold, pred, norm stuff, the original lls, + result_dict = { + **({"acc": acc} if "acc" in use_metric else {}), + **({"f1": (gold, pred)} if "f1" in use_metric else {}), + **({"mcc": (gold, pred)} if "mcc" in use_metric else {}), + **({"acc_norm": acc_norm} if "acc_norm" in use_metric else {}), + **({"exact_match": exact_match} if "exact_match" in use_metric else {}), + **( + {"brier_score": (gold, prob_norm)} + if "brier_score" in use_metric + else {} + ), + } + + if "acc_mutual_info" in use_metric: + lls_mutual_info = [ + ll_c - ll_u for ll_c, ll_u in zip(lls, lls_unconditional) + ] + acc_mutual_info = 1.0 if np.argmax(lls_mutual_info) == gold else 0.0 + result_dict["acc_mutual_info"] = acc_mutual_info + + elif self.OUTPUT_TYPE == "generate_until": + gold = self.doc_to_target(doc) + result = results[0] + if self.config.doc_to_choice is not None: + # If you set doc_to_choice, + # it assumes that doc_to_target returns a number. + choices = self.doc_to_choice(doc) + gold = choices[gold] + # we expect multiple_targets to be a list. + elif self.multiple_target: + gold = list(gold) + # TODO: handle this better + elif type(gold) is not type(result) and not ( + "bypass" in self._metric_fn_list.keys() or isinstance(result, list) + ): + # cast gold to the same type as result + gold = type(result)(gold) + + for metric in self._metric_fn_list.keys(): + if self.multiple_target: + # in the case where we have multiple targets, + # return true if any are true + # TODO: this may break for multipLe_target, non zero-or-1 metrics + scores = [] + if not isinstance(gold, list): + # sometimes, a multiple_target dataset has exceptions where one doc has only one string answer + # print(gold) + gold = [gold] + if metric == "exact_match": + result = [result for _ in range(len(gold))] + scores = self._metric_fn_list[metric]( + references=gold, + predictions=result, + **self._metric_fn_kwargs[metric], + )[metric] + result_score = 1.0 if scores > 0.0 else 0.0 + else: + for gold_option in gold: + try: + result_score = self._metric_fn_list[metric]( + references=[gold_option], + predictions=[result], + **self._metric_fn_kwargs[metric], + ) + except ( + TypeError + ): # TODO: this is hacky and I don't want to do it + result_score = self._metric_fn_list[metric]( + [gold_option, result] + ) + if isinstance(result_score, dict): + # TODO: this handles the case where HF evaluate returns a dict. + result_score = result_score[metric] + scores.append(result_score) + if any(scores): + result_score = 1.0 + else: + result_score = 0.0 + else: + try: + result_score = self._metric_fn_list[metric]( + references=[gold], + predictions=[result], + **self._metric_fn_kwargs[metric], + ) + except TypeError: # needed for now in order to use a different interface between our own metrics and HF Evaluate metrics + result_score = self._metric_fn_list[metric]([gold, result]) + if isinstance(result_score, dict): + # TODO: this handles the case where HF evaluate returns a dict. + # This allows for multiple metrics to be returned from the same function + for k, v in result_score.items(): + result_dict[k] = v + else: + result_dict[metric] = result_score + else: + raise ValueError( + f"Passed invalid output_type '{self.OUTPUT_TYPE}' ! Please use one of ", + "'loglikelihood', 'loglikelihood_rolling', 'generate_until' or 'multiple_choice'", + ) + + return result_dict + + def aggregation(self) -> dict: + return self._aggregation_list + + def higher_is_better(self) -> dict: + return self._higher_is_better + + def get_config(self, key: str) -> Any: + return getattr(self._config, key, None) + + @property + def task_name(self) -> Any: + return getattr(self.config, "task", None) + + def __repr__(self): + return ( + f"ConfigurableTask(task_name={getattr(self.config, 'task', None)}," + f"output_type={self.OUTPUT_TYPE}," + f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," + f"num_samples={len(self.eval_docs)})" + ) + + +class MultipleChoiceTask(Task): + OUTPUT_TYPE = "loglikelihood" + + def doc_to_target(self, doc: dict) -> str: + return " " + doc["choices"][doc["gold"]] + + def construct_requests(self, doc: dict, ctx: str, **kwargs) -> List[Instance]: + # TODO: add mutual info here? + return [ + Instance( + request_type="loglikelihood", + doc=doc, + arguments=(ctx, " {}".format(choice)), + idx=i, + **kwargs, + ) + for i, choice in enumerate(doc["choices"]) + ] + + def process_results(self, doc: dict, results: Iterable[Tuple[float, bool]]) -> dict: + results = [ + res[0] for res in results + ] # only retain loglikelihoods, discard is_greedy TODO: do we need is_greedy anywhere? + gold = doc["gold"] + + acc = 1.0 if np.argmax(results) == gold else 0.0 + completion_len = np.array([float(len(i)) for i in doc["choices"]]) + acc_norm = 1.0 if np.argmax(results / completion_len) == gold else 0.0 + + return { + "acc": acc, + "acc_norm": acc_norm, + } + + def higher_is_better(self) -> dict: + return { + "acc": True, + "acc_norm": True, + } + + def aggregation(self) -> dict: + return { + "acc": mean, + "acc_norm": mean, + } + + +class PerplexityTask(Task): + OUTPUT_TYPE = "loglikelihood_rolling" + + def has_training_docs(self) -> bool: + return False + + def fewshot_examples(self, k: int, rnd) -> List: + if k != 0: + raise ValueError( + "The number of fewshot examples must be 0 for perplexity tasks." + ) + return [] + + def fewshot_context(self, doc: dict, num_fewshot: int) -> Literal[""]: + if num_fewshot != 0: + raise ValueError( + "The number of fewshot examples must be 0 for perplexity tasks." + ) + + return "" + + def higher_is_better(self) -> dict: + return { + "word_perplexity": False, + "byte_perplexity": False, + "bits_per_byte": False, + } + + def doc_to_decontamination_query(self, doc): + return doc + + def doc_to_text(self, doc) -> str: + return "" + + def doc_to_target(self, doc): + return doc + + def construct_requests(self, doc: dict, ctx: Optional[str], **kwargs): + if bool(ctx): + raise ValueError + + return Instance( + request_type=self.OUTPUT_TYPE, + doc=doc, + arguments=(self.doc_to_target(doc),), + idx=0, + **kwargs, + ) + + def process_results(self, doc: dict, results: Tuple[float]) -> dict: + (loglikelihood,) = results + words = self.count_words(self.doc_to_target(doc)) + bytes_ = self.count_bytes(self.doc_to_target(doc)) + return { + "word_perplexity": (loglikelihood, words), + "byte_perplexity": (loglikelihood, bytes_), + "bits_per_byte": (loglikelihood, bytes_), + } + + def aggregation(self) -> dict: + return { + "word_perplexity": weighted_perplexity, + "byte_perplexity": weighted_perplexity, + "bits_per_byte": bits_per_byte, + } + + @classmethod + def count_bytes(cls, doc) -> int: + return len(doc.encode("utf-8")) + + @classmethod + def count_words(cls, doc) -> int: + """Downstream tasks with custom word boundaries should override this!""" + return len(re.split(r"\s+", doc)) diff --git a/lm-evaluation-harness/lm_eval/caching/__init__.py b/lm-evaluation-harness/lm_eval/caching/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-evaluation-harness/lm_eval/caching/__pycache__/__init__.cpython-310.pyc b/lm-evaluation-harness/lm_eval/caching/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59a3935f63049d2bf8dfc0e9206be21e1f478e65 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/caching/__pycache__/__init__.cpython-310.pyc differ diff --git a/lm-evaluation-harness/lm_eval/caching/__pycache__/__init__.cpython-311.pyc b/lm-evaluation-harness/lm_eval/caching/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29472a0e6a6376d8b0f780675b1dc0104b36dec2 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/caching/__pycache__/__init__.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/caching/__pycache__/cache.cpython-311.pyc b/lm-evaluation-harness/lm_eval/caching/__pycache__/cache.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7554ac105ae1911cc4ecc83888814d3b5c015908 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/caching/__pycache__/cache.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/decontamination/__init__.py b/lm-evaluation-harness/lm_eval/decontamination/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-evaluation-harness/lm_eval/decontamination/decontaminate.py b/lm-evaluation-harness/lm_eval/decontamination/decontaminate.py new file mode 100644 index 0000000000000000000000000000000000000000..2d1250d39bf7cd0272e412452d970ec7c52992c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/decontamination/decontaminate.py @@ -0,0 +1,166 @@ +import collections +import glob +import json +import os +import pickle +import random +import time + +from .archiver import ZStdTextReader +from .janitor import Janitor, word_ngrams + + +# Was used for testing the evaluator decoupled from the full logic below +def get_train_overlap_stub(docs: dict, ngrams_path: str, ngrams_n_size: str): + simulated_overlap = 0.1 + contaminated = int(len(docs) * simulated_overlap) + return random.sample(range(len(docs)), contaminated) + + +# Returns a dictionary containing all overlapping documents in each +# task. In the standard use case, an overlap occurs when any of the 13-grams +# found in the task document exist in the training set documents. +# +# To generate 13-grams for the pile see scripts/clean_training_data. The final output of these +# scripts are an info.json file containing the n_gram_size (13) and a bunch of "ngrams_{x}.bkt.txt.sorted.zst" +# files. These should exist in the "ngrams_path" provided to this function. + + +# Algorithm: +# 1. Build lookups for each dataset {ngram: list(document_ids)} +# 2. Merge into an overall lookup {ngram: [(task_name, task_set, doc_ids),]} +# 3. Full scan the 13-grams from the training set against the merged lookup, +# saving matches in the "duplicates" dictionary {(task_name, task_set): set(doc_ids)} +# 4. Strip the task_set from the dictionary keys and return +# +# We cache the task+set lookups as well as the overlaps. +def get_train_overlap(docs_by_task_set: dict, ngrams_path: str, limit: int) -> dict: + # return get_train_overlap_stub(docs, ngrams_path, ngrams_n_size) + + info_dict_path = os.path.join(ngrams_path, "info.json") + info_dict = json.load(open(info_dict_path, "r", encoding="utf-8")) + ngrams_n_size = info_dict["ngram_size"] + + janitor = Janitor() + + # Build lookup for each dataset first in case we use different task combinations later + print("Building Lookups...") + start = time.perf_counter() + + def get_overlaps_dump_path(task_name, task_set, ngrams_n_size, limit) -> str: + return f"data/{task_name}/{task_set}_{ngrams_n_size}grams_limit{limit}.overlaps" + + lookups = {} + duplicates = {} # (task_name, task_set): set(doc_ids)} + sets_to_decontaminate = len(docs_by_task_set.keys()) + + for (task_name, task_set), docs in docs_by_task_set.items(): + if not os.path.exists(f"data/{task_name}"): + os.mkdir(f"data/{task_name}") + + # Check if we've decontaminated this combination before + overlaps_dump_path = get_overlaps_dump_path( + task_name, task_set, ngrams_n_size, limit + ) + if os.path.exists(overlaps_dump_path): + duplicates[(task_name, task_set)] = pickle.load( + open(overlaps_dump_path, "rb") + ) + sets_to_decontaminate -= 1 + continue + else: + duplicates[(task_name, task_set)] = set() + + # Build/load the task lookup {ngram: set(documents)}. + task_set_lookup_path = ( + f"data/{task_name}/{task_set}_{ngrams_n_size}grams_limit{limit}.lookup" + ) + if os.path.exists(task_set_lookup_path): + print(f"{task_set_lookup_path} available, loading...") + lookups[(task_name, task_set)] = pickle.load( + open(task_set_lookup_path, "rb") + ) + else: + print(f"{task_set_lookup_path} not available, building...") + lookup = collections.defaultdict(set) + + for doc_id, document in enumerate(docs): + ngrams = word_ngrams(janitor.normalize_string(document), ngrams_n_size) + for ngram in ngrams: + lookup[ngram].add(doc_id) + + pickle.dump(lookup, open(task_set_lookup_path, "wb")) + lookups[(task_name, task_set)] = lookup + + elapsed = time.perf_counter() - start + print(f"Building lookups took {elapsed:0.5f} seconds.") + + matched_ngrams = [] + + if sets_to_decontaminate > 0: + print("Merging lookups...") + start = time.perf_counter() + merged_lookup = collections.defaultdict(list) + for (task_name, task_set), lookup in lookups.items(): + for ngram, doc_ids in lookup.items(): + merged_lookup[ngram].append((task_name, task_set, doc_ids)) + + elapsed = time.perf_counter() - start + print(f"Merging lookups took {elapsed:0.5f} seconds.") + + print(f"{ngrams_n_size} grams files found in {ngrams_path}:") + files = glob.glob(os.path.join(ngrams_path, "*.sorted.zst")) + print(files) + + for file in files: + start = time.perf_counter() + print(f"Scanning {file}") + reader = ZStdTextReader(file) + total_ngrams = 0 + unique_ngrams = 0 + matching_unique = 0 + non_matching_unique = 0 + + current_ngram = "" + for line in reader.read_tqdm(): # Scan training set ngrams file + total_ngrams += 1 + [ngram, document_id] = line.rsplit(" ", 1) + if ( + ngram != current_ngram + ): # Only need to match the ngram once in training set + unique_ngrams += 1 + current_ngram = ngram + if ngram in merged_lookup: + matched_ngrams.append(ngram) # For logging + matching_unique += 1 + for task_name, task_set, doc_ids in merged_lookup[ngram]: + task_doc_set = duplicates[(task_name, task_set)] + for doc_id in doc_ids: # Record contamination across all relevant task/set combos + task_doc_set.add(doc_id) + del merged_lookup[ngram] # No point matching again + else: + non_matching_unique += 1 + + print(f"Total Ngrams: {total_ngrams}") + print(f"Unique Ngrams: {unique_ngrams}") + print(f"Unique Matching: {matching_unique}") + print(f"Unique Non Matching: {non_matching_unique}") + print("Matched ngrams:") + for ngram in matched_ngrams: + print(ngram) + + elapsed = time.perf_counter() - start + print(f"Read took {elapsed:0.5f} seconds.") + print(f"Speed: {(os.path.getsize(file) / 1000000.0) / elapsed}MB/second") + + print(duplicates) + + # Dump overlaps separately + for (task_name, task_set), doc_ids in duplicates.items(): + overlaps_dump_path = get_overlaps_dump_path( + task_name, task_set, ngrams_n_size, limit + ) + pickle.dump(doc_ids, open(overlaps_dump_path, "wb")) + + # Strip task set and return + return {task_name: doc_ids for (task_name, task_set), doc_ids in duplicates.items()} diff --git a/lm-evaluation-harness/lm_eval/decontamination/janitor.py b/lm-evaluation-harness/lm_eval/decontamination/janitor.py new file mode 100644 index 0000000000000000000000000000000000000000..cedf8a5717aa8156674836ba236fdcabf36e0487 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/decontamination/janitor.py @@ -0,0 +1,328 @@ +import pickle +import re +import string +import traceback +from typing import Iterator, List, Sequence, Tuple, TypeVar + + +# This is a cpp module. Compile janitor_util.cpp with: +# c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) janitor_util.cpp -o janitor_util$(python3-config --extension-suffix) -undefined dynamic_lookup +try: + import janitor_util + + JANITOR_CPP = True +except Exception: + print("WARNING: C++ module could not be loaded. Janitor running in python mode") + traceback.print_exc() + JANITOR_CPP = False + +T = TypeVar("T") + + +# Implementation from nltk source +# https://www.nltk.org/_modules/nltk/util.html +def form_ngrams(sequence: Iterator[T], n: int) -> Iterator[Tuple[T, ...]]: + history = [] + while n > 1: + # PEP 479, prevent RuntimeError from being raised when StopIteration bubbles out of generator + try: + next_item = next(sequence) + except StopIteration: + # no more data, terminate the generator + return + history.append(next_item) + n -= 1 + for item in sequence: + history.append(item) + yield tuple(history) + del history[0] + + +def word_ngrams(s: str, n: int) -> Iterator[str]: + """Splits a string into ngram words""" + tokens = s.split() # not a generator :( + ngram_seqs = form_ngrams(iter(tokens), n) + return (" ".join(ngram) for ngram in ngram_seqs) + + +# Does character sequences only - combined faster function to play around with later +# def word_ngrams_indices_combined(sequence, n): +# current_word = "" +# history = [] +# gap = False; +# start = 0 +# end = 0 +# for character in sequence: +# if character == " ": +# if not gap: +# gap = True +# history.append(current_word) +# end += len(current_word) - 1 +# current_word = "" +# if len(history) == n: +# yield (tuple(history), start, end) +# del history[0] +# start = end + 1 +# end = start +# else: +# gap = False +# current_word += character + + +# https://stackoverflow.com/questions/13734451/string-split-with-indices-in-python +def split_indices(s: str) -> Iterator[Tuple[str, Tuple[int, int]]]: + """Splits a string on whitespaces and records the indices of each in the original string. + @:return generator((word, (start_idx, end_idx)), ...) + """ + return ((m.group(0), (m.start(), m.end() - 1)) for m in re.finditer(r"\S+", s)) + + +def word_ngrams_indices(s: str, n: int) -> Iterator[Tuple[str, Tuple[int, int]]]: + """Splits a string into pairs of (ngram words, their start/end indices)""" + tokens_with_indices = split_indices(s) + + # Generator of ngrams of (word, idx_pairs) + # ( + # [(word, (start,end)), (word, (start, end))...], + # [(word, (start, end)), ...], + # ... + # ) + ngram_seqs_with_indices = form_ngrams(tokens_with_indices, n) + + # Generator of pairs of word and index ngrams + # ( + # ([word, word, ...], [(start,end), (start,end), ...]), + # ... + # ) + ngram_indices_pairs = ( + zip(*ngram_with_indices) for ngram_with_indices in ngram_seqs_with_indices + ) + + # Generator of ( (word_ngram, (start, end)), (word_ngram, start, end)), ...) + return ( + (" ".join(ngram_seq), (indices[0][0], indices[-1][1])) + for ngram_seq, indices in ngram_indices_pairs + ) + + +class Janitor: + # FIXME delete_chars: Should anything else go here? Special chars? + def __init__( + self, + ngram_n: int = 13, + window_to_remove: int = 200, + too_dirty_cutoff: int = 10, + minimum_slice_length: int = 200, + delete_chars: str = string.punctuation, + ) -> None: + self.ngram_n = ngram_n + self.window_to_remove = window_to_remove + self.too_dirty_cutoff = too_dirty_cutoff + self.minimum_slice_length = minimum_slice_length + self.delete_chars = delete_chars + + self.dirt_ngrams = set() + + # If in python, we'll translate uppercase to lowercase and delete naughty characters. + # This is fast by python standards + # https://stackoverflow.com/questions/638893/what-is-the-most-efficient-way-in-python-to-convert-a-string-to-all-lowercase-st + self.translation_table = str.maketrans( + string.ascii_lowercase + string.ascii_uppercase, # These characters + string.ascii_lowercase * 2, # Become these characters + self.delete_chars, # These are deleted + ) + + ############## + # I/O for saving contamination ngrams + ############## + + def save_contamination_ngrams(self, filename: str) -> None: + with open(filename, "wb") as fp: + pickle.dump(filename, fp) + + def load_contamination_ngrams(self, filename: str) -> None: + with open(filename, "rb") as fp: + self.dirt_ngrams = pickle.load(fp) + + ############## + # Call these :) + ############## + + def register_contaminant(self, dirt_string: str) -> None: + """Register a string as contamination to be removed, e.g. a test set + This breaks the dirt_string into ngrams to store for future cleaning""" + if JANITOR_CPP: + return self.register_contaminant_cpp(dirt_string) + else: + print("WARNING: Janitor running in python mode") + return self.register_contaminant_python(dirt_string) + + def clean(self, dirty_string: str) -> List[str]: + """Clean a string (e.g. a training set) by removing all ngrams previously + registered as contaminants. Returns a list of clean chunks, or empty if + the string was too dirty""" + if JANITOR_CPP: + return self.clean_cpp(dirty_string) + else: + print("WARNING: Janitor running in python mode") + return self.clean_python(dirty_string) + + def _split_chunks( + self, dirty_string: str, dirty_parts: Sequence[Tuple] + ) -> List[str]: + clean_chunks = [] + splice_idx = 0 + end = -1 + for i, (ngram, start, end) in enumerate(dirty_parts): + if i >= self.too_dirty_cutoff: + return [] + start = max(0, start - self.window_to_remove) + end = min(len(dirty_string), end + self.window_to_remove) + + if start - splice_idx > self.minimum_slice_length: + clean_chunks.append(dirty_string[splice_idx:start]) + splice_idx = end + + if end < len(dirty_string) - self.minimum_slice_length: + clean_chunks.append(dirty_string[end + 1 :]) + + return clean_chunks + + ############## + # Fast C++ + ############## + + def register_contaminant_cpp(self, dirt_string) -> None: + self.dirt_ngrams.update( + janitor_util.clean_ngram(dirt_string, self.delete_chars, self.ngram_n) + ) + + def clean_cpp(self, dirty_string: str) -> List[str]: + contamination_indices = janitor_util.clean_ngram_with_indices( + dirty_string, self.delete_chars, self.ngram_n + ) + return self._split_chunks(dirty_string, contamination_indices) + + ############## + # Slow python + ############## + + def normalize_string(self, s: str) -> str: + return s.translate(self.translation_table) + + def register_contaminant_python(self, dirt_string: str) -> None: + self.dirt_ngrams.update( + word_ngrams(self.normalize_string(dirt_string), self.ngram_n) + ) + + def clean_python(self, dirty_string: str) -> List[str]: + contamination_indices = ( + (None, *idx_pair) + for dirty_ngram, idx_pair in word_ngrams_indices(dirty_string, self.ngram_n) + if self.normalize_string(dirty_ngram) in self.dirt_ngrams + ) + return self._split_chunks(dirty_string, contamination_indices) + + +################################################################## +# Tests +################################################################# + +# def print_cpp(): +# source = """ ,, I'm a very !dirty,, ,, dirty boy. Clean me daddy. \n\nhe he he hehe heh. lastword """ * 2 + +# for i in range(1, 10, 2): +# pprint(janitor_util.clean_ngram(source, string.punctuation, i)) +# for ngram, start, end in \ +# janitor_util.clean_ngram_with_indices(source, string.punctuation, i): +# print(ngram, "\t", start, end, source[start:end].replace("\n", "\\n")) + + +# def test_cpp(): +# source = """ ,, I'm a very !dirty,, ,, dirty boy. Clean me daddy. \n\nhe he he hehe heh. lastword """ * 2 +# contaminant = "dirty boy. Clean he he" + +# jan_python = Janitor() +# jan_cpp = Janitor() + +# jan_python.register_contaminant_python(contaminant) +# jan_cpp.register_contaminant(contaminant) + +# assert jan_python.dirt_ngrams == jan_cpp.dirt_ngrams, (jan_python.dirt_ngrams, jan_cpp.dirt_ngrams) + +# assert jan_python.clean_python(source) == jan_cpp.clean(source), \ +# (jan_python.clean_python(source), jan_cpp.clean(source)) + +# print("Passed test, python==cpp") + + +# def benchmark(): +# # Download and put in data folder: enwik8 (100 MB) from https://cs.fit.edu/~mmahoney/compression/textdata.html +# setup = \ +# """ +# with open("data/enwik8", "r") as f: +# data = f.read() +# jan = Janitor(too_dirty_cutoff=1000) +# jan.register_contaminant(''' +# theories is that there is a connection between "geekdom" and autism. +# This is hinted, for instance, by a ''Wired Magazine'' article in 2001 entitled " +# The [[Geek]] Syndrome", which is a point argued by many in the autism rights +# movement{{ref|Wired}}. This article, many professionals assert, is just one example of +# the media's application of mental disease labels to what is actually variant normal behavior +# &mdash;they argue that shyness, lack of athletic ability or social skills, and intellectual +# interests, even when they seem unusual to others, are not in themselves signs of autism or +# Asperger's syndrome. Others assert that it is actually the medical profession which is applying +# mental disease labels to children who in the past would have simply been accepted as a little +# different or even labeled 'gifted'. See [[clinomorphism]] for further discussion of this issue. +# Due to the recent publicity surrounding autism and autis +# ultan Al Nahyan]] granted [[Petroleum]] concessions, and oil was first found in 1958. At first, +# oil money had a marginal impact. A few lowrise concete buildings were erected, and the first +# paved road was completed in 1961, but Sheikh Shakbut, uncertain whether the new oil royalties +# would last, took a cautious approach, preferring to save the revenue rather than investing it in +# development. His brother, [[Zayed bin Sultan Al Nahayan]], saw that oil wealth had the potential +# to transform Abu Dhabi. The ruling Al Nahayan family decided that Sheikh Zayed should replace his +# brother as Ruler and carry out his vision of developing the country. On [[August 6]], [[1966]], +# with the assistance of the British, Sheikh Zayed became the new ruler. See generally, Al-Fahim, M, +# ''From Rags to Riches: A Story of Abu Dhabi'', Chapter Six (London Centre of Arab Studies, 1995), +# ISBN 1 900404 00 1. With the announcement by Britain in 1968 that it would withdraw from the +# Gulf area by 1971, Sheikh Zayed became the main driving force behind the formation of the +# [[United Arab Emirates]]. After the Emirates gained independence in 1971, +# ''') +# """ + +# n = 1 +# print(f"Timing {n} run on 100 MB") +# print("Register contaminant") +# # print("\tPython", timeit.timeit("jan.register_contaminant_python(data)", setup=setup, globals=globals(), number=n)) +# print("\tCpp", timeit.timeit("jan.register_contaminant(data)", setup=setup, globals=globals(), number=n)) + +# print("Clean") +# # print("\tPython", timeit.timeit("jan.clean_python(data)", setup=setup, globals=globals(), number=n)) +# print("\tCpp", timeit.timeit("jan.clean(data)", setup=setup, globals=globals(), number=n)) + + +# def test_janitor_general(): +# source = """ ,, I'm a very !dirty,, ,, dirty boy. Clean me daddy. \n\nhe he he hehe heh. lastword """ * 2 +# contaminant = "dirty boy. Clean he he" + +# jan = Janitor(ngram_n=3) +# jan.register_contaminant(contaminant) +# cleaned = " ".join(jan.clean(source)) +# for contam in jan.dirt_ngrams: +# assert contam not in cleaned, contam + +# filename = "data/saved_contam" +# jan.save_contamination_ngrams(filename) + +# jan = Janitor(ngram_n=3) +# jan.load_contamination_ngrams(filename) +# cleaned = " ".join(jan.clean(source)) +# for contam in jan.dirt_ngrams: +# assert contam not in cleaned, contam + + +# if __name__ == "__main__": +# test() +# # print_cpp() +# # test_cpp() +# # benchmark() diff --git a/lm-evaluation-harness/lm_eval/evaluator.py b/lm-evaluation-harness/lm_eval/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..d22cf6a5088675ea9da2eb08797a39691fd4fbe5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/evaluator.py @@ -0,0 +1,765 @@ +import itertools +import json +import logging +import random +import time +from collections import defaultdict +from typing import TYPE_CHECKING, List, Optional, Union + +import numpy as np +import torch + +import lm_eval.api.metrics +import lm_eval.api.registry +import lm_eval.api.task +import lm_eval.models +from lm_eval.caching.cache import delete_cache +from lm_eval.evaluator_utils import ( + consolidate_group_results, + consolidate_results, + get_sample_size, + get_subtask_list, + get_task_list, + prepare_print_tasks, + print_writeout, + run_task_tests, +) +from lm_eval.loggers import EvaluationTracker +from lm_eval.loggers.utils import add_env_info, add_tokenizer_info, get_git_commit_hash +from lm_eval.tasks import TaskManager, get_task_dict +from lm_eval.utils import ( + handle_non_serializable, + hash_string, + positional_deprecated, + setup_logging, + simple_parse_args_string, +) + + +if TYPE_CHECKING: + from lm_eval.api.model import LM + from lm_eval.api.task import Task + +eval_logger = logging.getLogger(__name__) + + +@positional_deprecated +def simple_evaluate( + model, + model_args: Optional[Union[str, dict]] = None, + tasks: Optional[List[Union[str, dict, object]]] = None, + num_fewshot: Optional[int] = None, + batch_size: Optional[Union[int, str]] = None, + max_batch_size: Optional[int] = None, + device: Optional[str] = None, + use_cache: Optional[str] = None, + cache_requests: bool = False, + rewrite_requests_cache: bool = False, + delete_requests_cache: bool = False, + limit: Optional[Union[int, float]] = None, + samples: Optional[dict] = None, + bootstrap_iters: int = 100000, + check_integrity: bool = False, + write_out: bool = False, + log_samples: bool = True, + evaluation_tracker: Optional[EvaluationTracker] = None, + system_instruction: Optional[str] = None, + apply_chat_template: Union[bool, str] = False, + fewshot_as_multiturn: bool = False, + gen_kwargs: Union[str, dict, None] = None, + task_manager: Optional[TaskManager] = None, + verbosity=None, + predict_only: bool = False, + random_seed: int = 0, + numpy_random_seed: int = 1234, + torch_random_seed: int = 1234, + fewshot_random_seed: int = 1234, + confirm_run_unsafe_code: bool = False, + metadata: Optional[dict] = None, +): + """Instantiate and evaluate a model on a list of tasks. + + :param model: Union[str, LM] + Name of model or LM object, see lm_eval.models.get_model + :param model_args: Optional[str, dict] + String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object. + Ignored if `model` argument is a LM object. + :param tasks: list[Union[str, dict, Task]] + List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise. + :param num_fewshot: int + Number of examples in few-shot context + :param batch_size: int or str, optional + Batch size for model + :param max_batch_size: int, optional + Maximal batch size to try with automatic batch size detection + :param device: str, optional + PyTorch device (e.g. "cpu" or "cuda:0") for running models + :param use_cache: str, optional + A path to a sqlite db file for caching model responses. `None` if not caching. + :param cache_requests: bool, optional + Speed up evaluation by caching the building of dataset requests. `None` if not caching. + :param rewrite_requests_cache: bool, optional + Rewrites all the request cache if set to `True`. `None` if not desired. + :param delete_requests_cache: bool, optional + Deletes all the request cache if set to `True`. `None` if not desired. + :param limit: int or float, optional + Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples. + :param samples: dictionary, optional + Dictionary indicating which examples should be tested in each task, e.g., {"mmlu_astronomy":[0,3,6],"mmlu_anatomy":[1,4,7,10]}. + :param bootstrap_iters: + Number of iterations for bootstrap statistics, used when calculating stderrs. set to 0 for no stderr calculations to be performed. + :param check_integrity: bool + Whether to run the relevant part of the test suite for the tasks + :param write_out: bool + If True, write out an example document and model input for checking task integrity + :param log_samples: bool + If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis + :param system_instruction: str + System instruction to be applied to the prompt + :param apply_chat_template: Union[bool, str] + Specifies whether to apply a chat template to the prompt. + - If set to True, the default chat template is applied. + - If set to a string, applies the specified chat template by name. + Defaults to False (no chat template applied). + :param fewshot_as_multiturn: bool + Whether to provide the fewshot examples as a multiturn conversation or a single user turn. + :param gen_kwargs: dict or comma-separated string + Arguments for model generation + Ignored for all tasks with loglikelihood output_type + :param verbosity: str + Verbosity level for logging + :param predict_only: bool + If true only model outputs will be generated and returned. Metrics will not be evaluated + :param random_seed: int + Random seed for python's random module. If set to None, the seed will not be set. + :param numpy_random_seed: int + Random seed for numpy. If set to None, the seed will not be set. + :param torch_random_seed: int + Random seed for torch. If set to None, the seed will not be set. + :param fewshot_random_seed: int + Random seed for fewshot sampler random generator. If set to None, the seed of generator will be set to None. + :param metadata: dict + Additional metadata to be added to the task manager. Will get passed to the download function of the task. + + return + Dictionary of results + """ + if verbosity is not None: + setup_logging(verbosity=verbosity) + start_date = time.time() + + if limit is not None and samples is not None: + raise ValueError( + "Either 'limit' or 'samples' must be None, but both are not None." + ) + + if ( + (isinstance(model_args, str) and "inst" in model_args.lower()) + or ( + isinstance(model_args, dict) + and any("inst" in str(v).lower() for v in model_args.values()) + ) + ) and not apply_chat_template: + eval_logger.warning( + "Model appears to be an instruct variant but chat template is not applied. Recommend setting `apply_chat_template` (optionally `fewshot_as_multiturn`)." + ) + + if delete_requests_cache: + eval_logger.info("Deleting requests cache...") + delete_cache() + + seed_message = [] + if random_seed is not None: + # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412 + seed_message.append(f"Setting random seed to {random_seed}") + random.seed(random_seed) + + if numpy_random_seed is not None: + seed_message.append(f"Setting numpy seed to {numpy_random_seed}") + np.random.seed(numpy_random_seed) + + if torch_random_seed is not None: + seed_message.append(f"Setting torch manual seed to {torch_random_seed}") + torch.manual_seed(torch_random_seed) + + if fewshot_random_seed is not None: + seed_message.append(f"Setting fewshot manual seed to {fewshot_random_seed}") + + if seed_message: + eval_logger.info(" | ".join(seed_message)) + + if tasks is None: + tasks = [] + if len(tasks) == 0: + raise ValueError( + "No tasks specified, or no tasks found. Please verify the task names." + ) + + if gen_kwargs is not None: + if isinstance(gen_kwargs, str): + gen_kwargs = simple_parse_args_string(gen_kwargs) + eval_logger.warning( + f"generation_kwargs: {gen_kwargs} specified through cli, these settings will update set parameters in yaml tasks. " + "Ensure 'do_sample=True' for non-greedy decoding!" + ) + if not gen_kwargs: + gen_kwargs = None + + if isinstance(model, str): + if model_args is None: + eval_logger.warning("model_args not specified. Using defaults.") + model_args = "" + + if isinstance(model_args, dict): + eval_logger.info( + f"Initializing {model} model, with arguments: {model_args}" + ) + lm = lm_eval.api.registry.get_model(model).create_from_arg_obj( + model_args, + { + "batch_size": batch_size, + "max_batch_size": max_batch_size, + "device": device, + }, + ) + + else: + eval_logger.info( + f"Initializing {model} model, with arguments: {simple_parse_args_string(model_args)}" + ) + lm = lm_eval.api.registry.get_model(model).create_from_arg_string( + model_args, + { + "batch_size": batch_size, + "max_batch_size": max_batch_size, + "device": device, + }, + ) + else: + if not isinstance(model, lm_eval.api.model.LM): + raise TypeError( + f"The value of `model` passed to simple_evaluate() was of type {type(model)}, but is required to be a subclass of lm_eval.api.model.LM . This may be because you are passing an initialized Hugging Face PreTrainedModel without having wrapped it in `lm_eval.models.huggingface.HFLM(pretrained=my_model)` first." + ) + eval_logger.info("Using pre-initialized model") + lm = model + + if use_cache is not None: + eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}") + lm = lm_eval.api.model.CachingLM( + lm, + use_cache + # each rank receives a different cache db. + # necessary to avoid multiple writes to cache at once + + "_rank" + + str(lm.rank) + + ".db", + ) + + if task_manager is None: + metadata = ( + simple_parse_args_string(model_args) + if isinstance(model_args, str) + else model_args + if isinstance(model_args, dict) + else {} + ) | (metadata or {}) + task_manager = TaskManager(metadata=metadata) + + task_dict = get_task_dict( + tasks, + task_manager, + ) + + # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups. + # (setting of num_fewshot ; bypassing metric calculation ; setting fewshot seed) + def _adjust_config(task_dict): + adjusted_task_dict = {} + for task_name, task_obj in task_dict.items(): + if isinstance(task_obj, dict): + adjusted_task_dict = { + **adjusted_task_dict, + **{task_name: _adjust_config(task_obj)}, + } + + else: + if task_obj.get_config("output_type") == "generate_until": + if gen_kwargs is not None: + task_obj.set_config( + key="generation_kwargs", value=gen_kwargs, update=True + ) + eval_logger.info( + f"{task_obj.config.task}: Using gen_kwargs: {task_obj.config.generation_kwargs}" + ) + + if predict_only: + eval_logger.info( + f"Processing {task_name} in output-only mode. Metrics will not be calculated!" + ) + # we have to change the class properties post-hoc. This is pretty hacky. + task_obj.override_metric(metric_name="bypass") + + # override tasks' fewshot values to the provided num_fewshot arg value + # except if tasks have it set to 0 manually in their configs--then we should never overwrite that + if num_fewshot is not None: + if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0: + eval_logger.info( + f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored." + ) + else: + eval_logger.warning( + f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}" + ) + task_obj.set_config(key="num_fewshot", value=num_fewshot) + else: + # if num_fewshot not provided, and the task does not define a default one, default to 0 + if ( + default_num_fewshot := task_obj.get_config("num_fewshot") + ) is None: + task_obj.set_config(key="num_fewshot", value=0) + # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file) + task_obj.set_fewshot_seed(seed=fewshot_random_seed) + + adjusted_task_dict[task_name] = task_obj + + return adjusted_task_dict + + task_dict = _adjust_config(task_dict) + + if check_integrity: + run_task_tests(task_list=tasks) + + if evaluation_tracker is not None: + evaluation_tracker.general_config_tracker.log_experiment_args( + model_source=model, + model_args=model_args, + system_instruction=system_instruction, + chat_template=lm.chat_template(apply_chat_template) + if apply_chat_template + else None, + fewshot_as_multiturn=fewshot_as_multiturn, + ) + + results = evaluate( + lm=lm, + task_dict=task_dict, + limit=limit, + samples=samples, + cache_requests=cache_requests, + rewrite_requests_cache=rewrite_requests_cache, + bootstrap_iters=bootstrap_iters, + write_out=write_out, + log_samples=True if predict_only else log_samples, + system_instruction=system_instruction, + apply_chat_template=apply_chat_template, + fewshot_as_multiturn=fewshot_as_multiturn, + verbosity=verbosity, + confirm_run_unsafe_code=confirm_run_unsafe_code, + ) + if verbosity is not None: + setup_logging(verbosity=verbosity) + + if lm.rank == 0: + if isinstance(model, str): + model_name = model + elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"): + model_name = model.config._name_or_path + else: + model_name = type(model).__name__ + + # add info about the model and few shot config + results["config"] = { + "model": model_name, + "model_args": model_args, + } + # add more detailed model info if available + if isinstance(lm, lm_eval.models.huggingface.HFLM): + results["config"].update(lm.get_model_info()) + # add info about execution + results["config"].update( + { + "batch_size": batch_size, + "batch_sizes": ( + list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else [] + ), + "device": device, + "use_cache": use_cache, + "limit": limit, + "bootstrap_iters": bootstrap_iters, + "gen_kwargs": gen_kwargs, + "random_seed": random_seed, + "numpy_seed": numpy_random_seed, + "torch_seed": torch_random_seed, + "fewshot_seed": fewshot_random_seed, + } + ) + results["git_hash"] = get_git_commit_hash() + results["date"] = start_date + add_env_info(results) # additional environment info to results + add_tokenizer_info(results, lm) # additional info about tokenizer + return results + else: + return None + + +@positional_deprecated +def evaluate( + lm: "LM", + task_dict, + limit: Optional[int] = None, + samples: Optional[dict] = None, + cache_requests: bool = False, + rewrite_requests_cache: bool = False, + bootstrap_iters: Optional[int] = 100000, + write_out: bool = False, + log_samples: bool = True, + system_instruction: Optional[str] = None, + apply_chat_template: Union[bool, str] = False, + fewshot_as_multiturn: bool = False, + verbosity: str = "INFO", + confirm_run_unsafe_code: bool = False, +): + """Instantiate and evaluate a model on a list of tasks. + + :param lm: obj + Language Model + :param task_dict: dict[str, Task] + Dictionary of tasks. Tasks will be taken to have name type(task).config.task . + :param limit: int, optional + Limit the number of examples per task (only use this for testing) + :param samples: dictionary, optional + Dictionary indicating which examples should be tested in each task, e.g., {"mmlu_astronomy":[0,3,6],"mmlu_anatomy":[1,4,7,10]}. + :param cache_requests: bool, optional + Speed up evaluation by caching the building of dataset requests. + :param rewrite_requests_cache: bool, optional + Rewrites all the request cache if set to `True`. + :param bootstrap_iters: + Number of iterations for bootstrap statistics, used when calculating stderr. Set to 0 for skipping all stderr calculations. + :param write_out: bool + If True, write out an example document and model input for checking task integrity + :param log_samples: bool + If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis + :param system_instruction: str + System instruction to be applied to the prompt + :param apply_chat_template: Union[bool, str] + Specifies whether to apply a chat template to the prompt. + - If set to True, the default chat template is applied. + - If set to a string, applies the specified chat template by name. + Defaults to False (no chat template applied). + :param fewshot_as_multiturn: bool + Whether to provide the fewshot examples as a multiturn conversation or a single user turn. + :param verbosity: str + Verbosity level for logging + :param confirm_run_unsafe_code: bool + Whether to confirm running tasks marked as unsafe. + :return + Dictionary of results + """ + + if limit is not None and samples is not None: + raise ValueError( + "Either 'limit' or 'samples' must be None, but both are not None." + ) + if samples is not None: + eval_logger.info(f"Evaluating examples for tasks {list(samples.keys())}") + if apply_chat_template: + eval_logger.warning( + "Chat template formatting change affects loglikelihood and multiple-choice tasks. See docs/chat-template-readme.md for details." + ) + # tracks all Instances/requests a model must generate output on. + requests = defaultdict(list) + # stores the amount to pad out reqs per req. type so that + # number of fwd passes per distributed rank is equal + padding_requests = defaultdict(int) + + # get lists of group hierarchy and each type of request + eval_tasks = get_task_list(task_dict) + if not log_samples: + if not all( + "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys() + for task_output in eval_tasks + ): + raise ValueError("log_samples must be True for 'bypass' metric-only tasks") + + # validation checks: + # 1.are we running multimodal task <-> non-multimodal model class, or vice-versa. + # 2.are we running code that is marked as unsafe. + incompatible_tasks = [] + for task_output in eval_tasks: + task: Task = task_output.task + + if getattr(task, "MULTIMODAL", False) and not getattr(lm, "MULTIMODAL", False): + incompatible_tasks.append(task_output.task_name) + elif getattr(task, "UNSAFE_CODE", False) and not confirm_run_unsafe_code: + raise ValueError( + f"Attempted to run task: {task_output.task_name} which is marked as unsafe. Set confirm_run_unsafe_code=True to run this task." + ) + if len(incompatible_tasks) > 0: + if not getattr(lm, "MULTIMODAL", False): + raise ValueError( + f"Attempted to run tasks: {incompatible_tasks} which require multimodal input, but the selected model type does not currently implement this. Multimodal support is currently restricted to the ['hf-multimodal', 'vllm-vlm'] model type." + ) + # end validation check + + # Cache the limit arg. + limit_arg = limit + limits = [] + for task_output in eval_tasks: + task: Task = task_output.task + + limit = get_sample_size(task, limit_arg) + limits.append(limit) + task.build_all_requests( + limit=limit, + samples=samples.get(task_output.task_name, None) + if samples is not None + else samples, + rank=lm.rank, + world_size=lm.world_size, + cache_requests=cache_requests, + rewrite_requests_cache=rewrite_requests_cache, + system_instruction=system_instruction, + apply_chat_template=bool(apply_chat_template), + fewshot_as_multiturn=fewshot_as_multiturn, + chat_template=getattr(lm, "apply_chat_template") + if apply_chat_template + else None, + tokenizer_name=getattr(lm, "tokenizer_name", "") + if apply_chat_template + else "", + ) + eval_logger.debug( + f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}" + ) + if write_out: + print_writeout(task) + # aggregate Instances by LM method requested to get output. + for instance in task.instances: + reqtype = instance.request_type + requests[reqtype].append(instance) + + if lm.world_size > 1: + instances_rnk = torch.tensor(len(task._instances), device=lm.device) + gathered_item = ( + lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist() + ) + # "multiple_choice" task types dispatch (several) "loglikelihood" request types + reqtype = ( + "loglikelihood" + if task.OUTPUT_TYPE == "multiple_choice" + else task.OUTPUT_TYPE + ) + # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks) + numpad = max(gathered_item) - gathered_item[lm.rank] + # todo: may not account for padding in cases like SquadV2 which has multiple req types + padding_requests[reqtype] += numpad + + ### Run LM on inputs, get all outputs ### + # execute each type of request + for reqtype, reqs in requests.items(): + eval_logger.info(f"Running {reqtype} requests") + # create `K` copies of each request `req` based off `K = req.repeats` + cloned_reqs = [] + for req in reqs: + cloned_reqs.extend([req] * req.repeats) + + if (lm.world_size > 1) and (padding_requests[reqtype] > 0): + for _ in range(padding_requests[reqtype]): + cloned_reqs.extend([req] * req.repeats) + + # run requests through model + resps = getattr(lm, reqtype)(cloned_reqs) + + # put responses from model into a list of length K for each request. + for x, req in zip(resps, cloned_reqs): + req.resps.append(x) + + if lm.world_size > 1: + lm.accelerator.wait_for_everyone() + + RANK = lm.rank + WORLD_SIZE = lm.world_size + ### Postprocess outputs ### + # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately) + for task_output, limit in zip(eval_tasks, limits): + task = task_output.task + task.apply_filters() + + ### Collect values of metrics on all datapoints ### + # # unpack results and sort back in order and return control to Task + # TODO: make it possible to use a different metric per filter + # Pre-process task.instances to group by doc_id + instances_by_doc_id = defaultdict(list) + for instance in task.instances: + instances_by_doc_id[instance.doc_id].append(instance) + # Sort instances within each group + for instances in instances_by_doc_id.values(): + instances.sort(key=lambda x: x.idx) + # iterate over different filters used + for filter_key in task.instances[0].filtered_resps.keys(): + indices = ( + samples.get(task_output.task_name, None) + if samples is not None + else None + ) + doc_iterator = task.doc_iterator( + rank=RANK, + limit=limit, + world_size=WORLD_SIZE, + samples=indices, + ) + for doc_id, doc in doc_iterator: + if indices: + doc_id_true = indices[doc_id] + else: + doc_id_true = doc_id + requests = instances_by_doc_id[doc_id] + metrics = task.process_results( + doc, [req.filtered_resps[filter_key] for req in requests] + ) + if log_samples: + target = task.doc_to_target(doc) + example = { + "doc_id": doc_id_true, + "doc": doc, + "target": target, + "arguments": [req.args for req in requests], + "resps": [req.resps for req in requests], + "filtered_resps": [ + req.filtered_resps[filter_key] for req in requests + ], + "filter": filter_key, + "metrics": list(metrics.keys()), + "doc_hash": hash_string( + json.dumps( + requests[0].doc, + indent=2, + default=handle_non_serializable, + ensure_ascii=False, + ) + ), + "prompt_hash": hash_string(requests[0].arguments[0]), + "target_hash": hash_string(str(target)), + } + example.update(metrics) + task_output.logged_samples.append(example) + for metric, value in metrics.items(): + task_output.sample_metrics[(metric, filter_key)].append(value) + + if WORLD_SIZE > 1: + # if multigpu, then gather data across all ranks to rank 0 + # first gather logged samples across all ranks + for task_output in eval_tasks: + if log_samples: + # for task_name, task_samples in list(samples.items()): + full_samples = [None] * WORLD_SIZE if RANK == 0 else None + torch.distributed.gather_object( + obj=task_output.logged_samples, + object_gather_list=full_samples, + dst=0, + ) + + if RANK == 0: + task_output.logged_samples = list( + itertools.chain.from_iterable(full_samples) + ) + + # then collect metrics across all ranks + for metrics in task_output.sample_metrics: + metric_list = [None] * WORLD_SIZE if RANK == 0 else None + torch.distributed.gather_object( + obj=task_output.sample_metrics[metrics], + object_gather_list=metric_list, + dst=0, + ) + if RANK == 0: + task_output.sample_metrics[metrics] = list( + itertools.chain.from_iterable(metric_list) + ) + + if RANK == 0: + ### Aggregate results over all datapoints ### + # aggregate results ; run bootstrap CIs + for task_output in eval_tasks: + task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters) + ( + results, + samples, + configs, + versions, + num_fewshot, + higher_is_better, + ) = consolidate_results(eval_tasks) + + ### Calculate group metrics ### + if bool(results): + results, versions, show_group_table, *_ = consolidate_group_results( + results, versions, task_dict + ) + + results_agg, group_agg = prepare_print_tasks(task_dict, results) + subtask_list = get_subtask_list(task_dict) + + # collect all higher_is_better values for metrics + # in the group's subtasks. + # TODO: clean this up ; unify with the below metric_list loop? + _higher_is_better = {} + for group, task_list in subtask_list.items(): + if ( + len(task_list) != 0 + ): # subtask list will list "task_name": [] for solo tasks + for task in task_list: + for m, h in higher_is_better[task].items(): + if m not in _higher_is_better.keys(): + _higher_is_better[m] = h + + if ( + m in _higher_is_better + and _higher_is_better[m] is not None + and _higher_is_better[m] != h + ): + eval_logger.warning( + f"Higher_is_better values for metric {m} in group {group} are not consistent. Defaulting to None." + ) + _higher_is_better[m] = None + higher_is_better[group] = _higher_is_better + + results_dict = { + "results": dict(results_agg.items()), + **( + {"groups": dict(group_agg.items())} + if (bool(group_agg) & show_group_table) + else {} + ), + "group_subtasks": dict(reversed(subtask_list.items())), + "configs": dict(sorted(configs.items())), + "versions": dict(sorted(versions.items())), + "n-shot": dict(sorted(num_fewshot.items())), + "higher_is_better": dict(sorted(higher_is_better.items())), + "n-samples": { + task_output.task_name: { + "original": len(task_output.task.eval_docs), + "effective": min( + limit if limit else len(task_output.task.eval_docs), + len(task_output.task.eval_docs), + ), + } + for task_output, limit in zip(eval_tasks, limits) + }, + } + if log_samples: + results_dict["samples"] = dict(samples) + + return results_dict + + else: + return None + + +def request_caching_arg_to_dict(cache_requests: str) -> dict: + request_caching_args = { + "cache_requests": cache_requests in {"true", "refresh"}, + "rewrite_requests_cache": cache_requests == "refresh", + "delete_requests_cache": cache_requests == "delete", + } + + return request_caching_args diff --git a/lm-evaluation-harness/lm_eval/evaluator_utils.py b/lm-evaluation-harness/lm_eval/evaluator_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0bd87b6c7a3923d7e2cdf71894ca87b10137ac9c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/evaluator_utils.py @@ -0,0 +1,554 @@ +import collections +import logging +import math +import pathlib +import sys +from typing import List, Optional, Tuple, Union + +from lm_eval.api.group import ConfigurableGroup +from lm_eval.api.metrics import ( + aggregate_subtask_metrics, + mean, + pooled_sample_stderr, + stderr_for_metric, +) +from lm_eval.api.task import Task +from lm_eval.utils import positional_deprecated + + +eval_logger = logging.getLogger(__name__) + + +class TaskOutput: + """ + Wrapper class for Task outputs.It contains various attributes and methods to manage and calculate metrics for the task. + + Attributes: + task (object): The task object. + task_name (str): The name of the task. + task_config (dict): The configuration of the task. + version (str): The version of the task. + group_name (str): The name of the task group. + n_shot (int): The number of shots for the task. + task_alias (str): The alias of the task. + group_alias (str): The alias of the task group. + is_group (bool): Indicates if the task is a group. + logged_samples (list): The list of logged samples. + sample_len (int): The length of the samples. + sample_metrics (defaultdict): The dictionary of samples' metrics. + agg_metrics (defaultdict): The dictionary of aggregate metrics. + + Methods: + from_taskdict(cls, task_name: str, task): + Creates a TaskOutput instance from a task dictionary. + + calculate_aggregate_metric(bootstrap_iters=100000) -> None: + Calculates the aggregate metrics for the task. + """ + + def __init__( + self, + task=None, + task_name=None, + task_config=None, + version=None, + group_name=None, + n_shot=None, + task_alias=None, + group_alias=None, + is_group=None, + ): + self.task = task + self.task_config = task_config + self.task_name = task_name + self.group_name = group_name + self.version = version + self.n_shot = n_shot + self.task_alias = task_alias + self.group_alias = group_alias + self.is_group = is_group + self.logged_samples = [] + self.sample_len = None + self.sample_metrics = collections.defaultdict(list) + self.agg_metrics = collections.defaultdict(list) + + @classmethod + def from_taskdict(cls, task_name: str, task): + if isinstance(task, tuple): + group_name, task = task + else: + group_name = None + if not task: + # these gets filtered out in get_task_list + # once they are added to group hierarchy + is_group = True + return cls( + task=task, task_name=task_name, is_group=is_group, group_name=group_name + ) + version = task.VERSION + task_config = dict(task.dump_config()) + if (n_shot := task_config.get("num_fewshot")) == 0: + n_shot = task_config.get("metadata", {}).get("num_fewshot", 0) + task_alias = task_config.get("alias") + group_alias = task_config.get("group_alias") + return cls( + task=task, + task_name=task_name, + task_config=task_config, + group_name=group_name, + version=version, + n_shot=n_shot, + task_alias=task_alias, + group_alias=group_alias, + ) + + def calculate_aggregate_metric(self, bootstrap_iters=100000) -> None: + for (metric, filter_key), items in self.sample_metrics.items(): + try: + agg_fn = self.task.aggregation()[metric] + except KeyError: + # This is when process results output an arbitrary metric + # TODO: Handle this better and allow other aggregate functions other than mean. + agg_fn = mean + metric_key = f"{metric},{filter_key}" + self.agg_metrics[metric_key] = agg_fn(items) + self.sample_len = len(items) # TODO: same sample size for each metric? + if isinstance(bootstrap_iters, int): + stderr_fn = stderr_for_metric( + metric=agg_fn, + bootstrap_iters=min(bootstrap_iters, 100) + if metric in ["bleu", "chrf", "ter"] + else bootstrap_iters, + ) + self.agg_metrics[f"{metric}_stderr,{filter_key}"] = ( + stderr_fn(items) if (stderr_fn and len(items) > 1) else "N/A" + ) + else: + raise ValueError( + f"Received bootstrap_iters '{bootstrap_iters}' but expected an integer. Set to 0 to turn off stderr calculations." + ) + + def __repr__(self): + return ( + f"TaskOutput(task_name={self.task_name}, " + f"group_name={self.group_name}, " + f"version={self.version}, " + f"n_shot={self.n_shot}, " + f"task_alias={self.task_alias}, " + f"group_alias={self.group_alias})" + ) + + +def get_task_list(task_dict: dict) -> List[TaskOutput]: + outputs = [] + for task_name, task_obj in task_dict.items(): + if isinstance(task_obj, dict): + _outputs = get_task_list(task_obj) + outputs.extend(_outputs) + else: + task_output = TaskOutput.from_taskdict(task_name, task_obj) + outputs.append(task_output) + + return outputs + + +def get_subtask_list(task_dict, task_root=None, depth=0): + subtask_list = {} + for group_obj, task_obj in task_dict.items(): + if isinstance(group_obj, ConfigurableGroup): + # group_name = group_obj.group_name + group_name = group_obj.group_name + else: + group_name = group_obj + if isinstance(task_obj, dict): + _subtask_list = get_subtask_list( + task_obj, task_root=group_name, depth=depth + 1 + ) + if task_root: + subtask_list.setdefault((task_root, depth), []).extend( + [ + _task + for (_task, _depth) in _subtask_list.keys() + if (_depth - 1) == depth + ] + ) + + subtask_list = {**subtask_list, **_subtask_list} + else: + if isinstance(task_obj, ConfigurableGroup): + # group_or_task_name = task_obj.group_name + group_or_task_name = task_obj.group_name + elif isinstance(task_obj, Task): + # group_or_task_name = task_obj.task_name + group_or_task_name = task_obj.task_name + + if task_root is None: + subtask_list.setdefault((group_or_task_name, depth), []) + else: + subtask_list.setdefault((task_root, depth), []).append( + group_or_task_name + ) + + if depth == 0: + _subtask_list = {} + for group_key, task_list in subtask_list.items(): + group_name, depth = group_key + _subtask_list[group_name] = task_list + subtask_list = _subtask_list + + return subtask_list + + +def print_writeout(task) -> None: + for inst in task.instances: + # print the prompt for the first few documents + if inst.doc_id < 1: + eval_logger.info( + f"Task: {task}; document {inst.doc_id}; context prompt (starting on next line):\ + \n{inst.args[0]}\n(end of prompt on previous line)\ntarget string or answer choice index (starting on next line):\n{task.doc_to_target(inst.doc)}\n(end of target on previous line)" + ) + eval_logger.info(f"Request: {str(inst)}") + + +def get_sample_size(task, limit: Optional[int]) -> Union[int, None]: + if limit is not None: + limit = ( + int(math.ceil(len(task.eval_docs) * limit)) if limit < 1.0 else int(limit) + ) + return limit + + +def prepare_print_tasks( + task_dict: dict, + results: dict, + task_depth=0, + group_depth=0, +) -> Tuple[dict, dict]: + """ + @param task_dict: Dictionary representing the group hierarchy of tasks. Each key is a group name and its + value is a list of task names. + @param results: Dictionary containing the results of each task. Each key is a + group name and its value is a dictionary of task results. + @param task_depth: The indentation level for printing the task + hierarchy. Default is 0. + @param group_depth: The indentation level for printing the group + hierarchy. Default is 0. + @return: A tuple of two dictionaries: results_agg and groups_agg. results_agg contains + aggregated results for each task, and groups_agg contains aggregated results for each group. + + Prepares the task hierarchy and aggregates the results for each task and group recursively for printing. + """ + + def _sort_task_dict(task_dict): + """ + Helper utility. Sorts the task dict at the current level of the hierarchy based on alphabetized task name. + Required so that we end up sorting within each sub-header correctly. + """ + + return dict( + sorted( + task_dict.items(), + key=lambda item: item[0].group_name + if isinstance(item[0], ConfigurableGroup) + else item[0], + ) + ) + + task_agg = collections.defaultdict(dict) + group_agg = collections.defaultdict(dict) + task_dict = _sort_task_dict(task_dict) + for task_or_group_name, task_or_group_obj in task_dict.items(): + tab_string = " " * task_depth + "- " if task_depth > 0 else "" + if isinstance(task_or_group_name, ConfigurableGroup): + # string_name = task_or_group_name.group_name + name = task_or_group_name.group_name + from_configurable_group = True + task_or_group_obj = _sort_task_dict(task_or_group_obj) + elif isinstance(task_or_group_name, str): + name = task_or_group_name + if isinstance(task_or_group_obj, Task): + # string_name = task_or_group_obj.task_name + name = task_or_group_obj.task_name + from_configurable_group = False + + task_agg[name] = results[name].copy() + if from_configurable_group: + if task_or_group_name.group_alias is not None: + alias = task_or_group_name.group_alias + else: + alias = task_or_group_name.group + else: + if "alias" in task_agg[name]: + alias = task_agg[name]["alias"] + else: + alias = name + + task_agg[name]["alias"] = tab_string + alias + if "samples" in task_agg[name]: + task_agg[name].pop("samples") + + if from_configurable_group and (" " not in results[name]): + group_tab_string = " " * group_depth + "- " if group_depth > 0 else "" + group_agg[name] = results[name].copy() + group_agg[name]["alias"] = group_tab_string + alias + if "samples" in group_agg[name]: + group_agg[name].pop("samples") + + if isinstance(task_or_group_obj, dict): + task_depth += 1 + group_depth += 1 + _task_agg, _group_agg = prepare_print_tasks( + task_or_group_obj, results, task_depth, group_depth + ) + task_agg = { + **task_agg, + **_task_agg, + } + group_agg = {**group_agg, **_group_agg} + task_depth -= 1 + group_depth -= 1 + return task_agg, group_agg + + +def consolidate_results( + eval_tasks: List[TaskOutput], +) -> Tuple[dict, dict, dict, dict, dict, dict]: + """ + @param eval_tasks: list(TaskOutput). + @return: A tuple containing the consolidated results, samples, configs, versions, and num_fewshot. + + Consolidates the results of multiple evaluation tasks into a single structure. + + The method iterates over each evaluation instance and extracts relevant information to create the consolidated + results structure. The consolidated results structure has the following properties: + + - results: A defaultdict with task names as keys and dictionaries as values. Each dictionary contains + metric/filter pairs as keys and corresponding metric values as values. The "alias" key is used to store task + aliases specified in the task configuration. + - samples: A defaultdict with task names as keys and lists of log samples as values. + - configs: A defaultdict with task names as keys and task configurations as values. + - versions: A defaultdict with task names as keys and task versions as values. + - num_fewshot: A defaultdict with task names as keys and number of few-shot samples as values. + - higher_is_better: A defaultdict with task names as keys and indicators of whether higher values are better + for each metric as values. + + The method then returns the consolidated results, samples, configs, versions, and num_fewshot as a tuple. + """ + # stores the final result for each task, for each metric/filter pair. + results = collections.defaultdict(dict) + # logs info about each document evaluated. + samples = collections.defaultdict(list) + # store num-fewshot value per task + num_fewshot = collections.defaultdict(int) + # Tracks the YAML configs of all chosen task + configs = collections.defaultdict(dict) + # Tracks each task's version. + versions = collections.defaultdict(dict) + # Track `higher_is_better` for each metric + higher_is_better = collections.defaultdict(dict) + + for task_output in eval_tasks: + if "task_alias" in (task_config := task_output.task_config): + results[task_output.task_name]["alias"] = task_config["task_alias"] + else: + results[task_output.task_name]["alias"] = task_output.task_name + if group_alias := task_output.group_alias: + if group_alias not in results and (group_name := task_output.group_name): + results[group_name]["alias"] = group_alias + num_fewshot[task_output.task_name] = task_output.n_shot + configs[task_output.task_name] = task_output.task_config + versions[task_output.task_name] = task_output.version + samples[task_output.task_name] = task_output.logged_samples + higher_is_better[task_output.task_name] = task_output.task.higher_is_better() + for (metric, filter_key), items in task_output.sample_metrics.items(): + metric_key = f"{metric},{filter_key}" + results[task_output.task_name][metric_key] = task_output.agg_metrics[ + metric_key + ] + results[task_output.task_name]["samples"] = task_output.sample_len + results[task_output.task_name][f"{metric}_stderr,{filter_key}"] = ( + task_output.agg_metrics[f"{metric}_stderr,{filter_key}"] + ) + return results, samples, configs, versions, num_fewshot, higher_is_better + + +def consolidate_group_results( + results, + versions, + task_dict, + task_root=None, + show_group_table=False, + task_aggregation_list=None, +) -> Tuple[dict, dict, bool, Union[None,]]: + """ + (Recursively) calculates groups' aggregated metrics and updates the results and versions dictionaries with this info. + + @return: a tuple [results, versions, show_group_table, task_aggregation_list] with formats described below: + + - results: A defaultdict with task names (and, after this function is called, group names of + groups that perform aggregation) as keys, and dictionaries with "alias" and metric,filter_name pairs as keys. + - versions: A defaultdict with task names (and, after this function is called, group names of + groups that perform aggregation) as keys, and float values representing the task or group's version if a version is specified. (defaulting to None). + - show_group_table: a boolean which is true if there exists a group that requires printing of its aggregated scores in a group table. + - task_aggregation_list: a defaultdict listing the subtasks to average over to produce a given group's end metric. + + The method then returns the updated results, versions, show_group_table, and task_aggregation_list as a tuple. + In the top-level invocation of this function, task_aggregation_list is ignored. + """ + if task_root is None: + task_root = {} + + if task_aggregation_list is None: + task_aggregation_list = {} + + for group_or_task, group_or_task_info in task_dict.items(): + # Convert to string + if isinstance(group_or_task, ConfigurableGroup): + group_config = group_or_task.config + group_or_task = group_or_task.group_name + else: + group_config = None + + if isinstance(group_or_task_info, Task): + if task_root: + task_aggregation_list.setdefault(task_root, []).append( + group_or_task_info.task_name + ) + else: + ( + results, + versions, + show_group_table, + _task_aggregation_list, + ) = consolidate_group_results( + results, + versions, + group_or_task_info, + group_or_task, + show_group_table, + task_aggregation_list, + ) + if task_root: + task_aggregation_list.setdefault(task_root, []).extend( + task_aggregation_list.get(group_or_task, []) + ) + + if (group_config is None) or ( + group_config["aggregate_metric_list"] is None + ): + results[group_or_task][" "] = " " + continue + + if "aggregate_metric_list" in group_config: + agg_metric_list = group_config["aggregate_metric_list"] + + show_group_table = show_group_table | bool( + group_config["aggregate_metric_list"] + ) + + task_list = _task_aggregation_list[group_or_task] + + metric_list = list( + { + key + for task in task_list + for key in results[task].keys() + if "_stderr" not in key and key not in ["task", "alias", "samples"] + } + ) + for metric in metric_list: + stderr = "_stderr,".join(metric.split(",")) + + # gather metrics, sizes, and stderrs from subtasks + metrics = [ + results[task][metric] + for task in task_list + if metric in results[task] + ] # TODO: copy? + stderrs = [ + results[task][stderr] + for task in task_list + if stderr in results[task] + ] + sizes = [ + results[task]["samples"] + for task in task_list + if metric in results[task] + ] + + for metric_config in agg_metric_list: + for filter_name in metric_config["filter_list"]: + if metric != ",".join([metric_config["metric"], filter_name]): + continue + + # compute group's pooled metric and stderr + if metric_config["aggregation"] == "mean": + aggregate_fn = aggregate_subtask_metrics + elif callable(metric_config["aggregation"]): + aggregate_fn = metric_config["aggregation"] + else: + raise ValueError( + f"Currently, only 'mean' is supported for automatically aggregating scores across groups' subtasks. Got '{metric_config['aggregation']}' for group '{group_or_task}'" + ) + + results[group_or_task][metric] = aggregate_fn( + metrics, + sizes, + metric_config["weight_by_size"], + ) + # TODO: calculate groups' metrics using arbitrary agg fns + if "N/A" in stderrs: + results[group_or_task][stderr] = "N/A" + else: + # NOTE: this assumes we are using the mean to aggregate. There are warnings about this elsewhere + results[group_or_task][stderr] = pooled_sample_stderr( + stderrs, sizes + ) + + results[group_or_task]["samples"] = sum(sizes) + group_metadata = group_config.get("metadata", None) + if group_metadata is not None: + versions[group_or_task] = group_metadata.get("version", None) + # print(results) + return results, versions, show_group_table, task_aggregation_list + + +@positional_deprecated +def find_test_root(start_path: pathlib.Path) -> pathlib.Path: + """ + Search upward in the directory tree to a maximum of three layers + to find and return the package root (containing the 'tests' folder) + """ + cur_path = start_path.resolve() + max_layers = 3 + for _ in range(max_layers): + if (cur_path / "tests" / "test_version_stable.py").exists(): + return cur_path + else: + cur_path = cur_path.parent.resolve() + raise FileNotFoundError( + f"Unable to find package root within {max_layers} upwards" + f"of {start_path}" + ) + + +@positional_deprecated +def run_task_tests(task_list: List[str]): + """ + Find the package root and run the tests for the given tasks + """ + import pytest + + package_root = find_test_root(start_path=pathlib.Path(__file__)) + task_string = " or ".join(task_list) + args = [ + f"{package_root}/tests/test_version_stable.py", + f"--rootdir={package_root}", + "-k", + f"{task_string}", + ] + sys.path.append(str(package_root)) + pytest_return_val = pytest.main(args) + if pytest_return_val: + raise ValueError( + f"Not all tests for the specified tasks ({task_list}) ran successfully! Error code: {pytest_return_val}" + ) diff --git a/lm-evaluation-harness/lm_eval/filters/__init__.py b/lm-evaluation-harness/lm_eval/filters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..be5c9d43624ea901cc578c65689be5bd263209a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/filters/__init__.py @@ -0,0 +1,25 @@ +from functools import partial +from typing import List + +from lm_eval.api.filter import FilterEnsemble +from lm_eval.api.registry import get_filter + +from . import custom, extraction, selection, transformation + + +def build_filter_ensemble( + filter_name: str, components: List[List[str]] +) -> FilterEnsemble: + """ + Create a filtering pipeline. + """ + filters = [] + for function, kwargs in components: + if kwargs is None: + kwargs = {} + # create a filter given its name in the registry + f = partial(get_filter(function), **kwargs) + # add the filter as a pipeline step + filters.append(f) + + return FilterEnsemble(name=filter_name, filters=filters) diff --git a/lm-evaluation-harness/lm_eval/filters/__pycache__/selection.cpython-311.pyc b/lm-evaluation-harness/lm_eval/filters/__pycache__/selection.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4489c0e8621e4ab52729fdb3b0159a1670aec238 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/filters/__pycache__/selection.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/filters/__pycache__/transformation.cpython-311.pyc b/lm-evaluation-harness/lm_eval/filters/__pycache__/transformation.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbcd15566e31c64a1af0da39ab28d4986fc02c11 Binary files /dev/null and b/lm-evaluation-harness/lm_eval/filters/__pycache__/transformation.cpython-311.pyc differ diff --git a/lm-evaluation-harness/lm_eval/filters/extraction.py b/lm-evaluation-harness/lm_eval/filters/extraction.py new file mode 100644 index 0000000000000000000000000000000000000000..22ca883a9d00b2156c6aedc5df7448879a03da65 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/filters/extraction.py @@ -0,0 +1,233 @@ +import re +import sys +import unicodedata + +from lm_eval.api.filter import Filter +from lm_eval.api.registry import register_filter + + +@register_filter("regex") +class RegexFilter(Filter): + """A filter that extracts values from text using regex pattern matching. + + This filter applies a regex pattern to each model response and extracts matched values. + If no match is found, returns a fallback value. Useful for extracting structured data + (like numbers) from unstructured model outputs. + """ + + def __init__( + self, + regex_pattern: str = r"#### (\-?[0-9\.\,]+)", + group_select: int = 0, + fallback: str = "[invalid]", + ) -> None: + """ + pass a string `regex` to run `re.compile(r"regex")` on. + `fallback` defines the output returned if no matches for the regex are located. + """ + self.regex_pattern = regex_pattern + self.regex = re.compile(regex_pattern) + self.group_select = group_select + self.fallback = fallback + + def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + # here, we assume we have a list, in which each element is + # a list of model responses for some particular input/target pair. + # so we process each of these (same input/target response sets) + # independently (and keep them a list.) + def filter_set(inst): + filtered = [] + for resp in inst: + match = self.regex.findall(resp) + if match: + match = match[self.group_select] + if isinstance(match, tuple): + match = [m for m in match if m] + if match: + match = match[0] + else: + match = self.fallback + match = match.strip() + else: + match = self.fallback + filtered.append(match) + return filtered + + filtered_resps = list(map(lambda x: filter_set(x), resps)) + return filtered_resps + + +@register_filter("regex_pos") +class POSFilter(Filter): + """ """ + + def __init__( + self, + regex_pattern: str = r"\['(.*?)'\]", + group_select=0, + fallback=None, + ) -> None: + """ + pass a string `regex` to run `re.compile(r"regex")` on. + `fallback` defines the output returned if no matches for the regex are located. + """ + if fallback is None: + fallback = ["invalid"] + self.regex_pattern = regex_pattern + self.regex = re.compile(regex_pattern) + self.group_select = group_select + self.fallback = fallback + + def apply(self, resps, docs): + def extract_tagged_tokens(text): + # Extract tagged tokens list from text input using regex + tokens = re.findall(r"\('([^']*)', '([^']*)'\)", text) + return [(token, pos) for token, pos in tokens] + + def extract_pos_tags(result): + pos_tags = [] + if isinstance(result, str): + result = extract_tagged_tokens(result) + pos_tags.extend(pos for _, pos in result) + return pos_tags if pos_tags else self.fallback + + def filter_set(inst): + filtered = [] + for resp in inst: + match = extract_pos_tags(resp) + filtered.append(match) + return filtered + + filtered_resps = map(lambda x: filter_set(x), resps) + + return filtered_resps + + +@register_filter("remove_whitespace") +class WhitespaceFilter(Filter): + """Filters out leading whitespace from responses.""" + + def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + def filter_set(inst): + filtered_resp = [] + for resp in inst: + resp = resp.lstrip() + filtered_resp.append(resp) + return filtered_resp + + filtered_resps = [filter_set(resp) for resp in resps] + + return filtered_resps + + +@register_filter("multi_choice_regex") +class MultiChoiceRegexFilter(RegexFilter): + """ + A filter used to extract a model's answer on multiple choice questions with + letter answers. assumes each document has a "choices" field + containing the list of answer choices and that the answer label symbols + are of the form (A), (B), (C), ... or A, B, C. + """ + + def __init__( + self, + regex_pattern: str = r"#### (\-?[0-9\.\,]+)", + group_select=0, + fallback: str = "[invalid]", + ignore_case=False, + ignore_punctuation=False, + regexes_to_ignore=None, + ) -> None: + """ + regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure + - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response. + - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices. + group_select: Selects the (group_select)th match from the findall result. + ignore_case: Ignores the case during step 1 matching + ignore_punctuation: Remove the punctuation during step 1 matching + regexes_to_ignore: Remove these regexes during step 1 matching + """ + super().__init__(regex_pattern, group_select, fallback) + self.ignore_case = ignore_case + self.ignore_punctuation = ignore_punctuation + self.regexes_to_ignore = regexes_to_ignore + + def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + # here, we assume we have a list, in which each element is + # a list of model responses for some particular input/target pair. + # so we process each of these (same input/target response sets) + # independently (and keep them a list.) + + def find_match(regex, resp, convert_dict={}): + match = regex.findall(resp) + if match: + match = match[self.group_select] + if isinstance(match, tuple): + match = [m for m in match if m][0] + match = match.strip() + if match and match in convert_dict: + match = convert_dict[match] + return match + + punct_tbl = dict.fromkeys( + i + for i in range(sys.maxunicode) + if unicodedata.category(chr(i)).startswith("P") + ) + + def filter_ignores(st): + if self.regexes_to_ignore is not None: + for s in self.regexes_to_ignore: + st = re.sub(s, "", st) + + if self.ignore_case: + st = st.lower() + + if self.ignore_punctuation: + # https://stackoverflow.com/a/266162 + st = st.translate(punct_tbl) + return st + + filtered_resps = [] + + for r, doc in zip(resps, docs): + fallback_regexes = [] + choice_to_alpha = {} + next_alpha = "A" + + without_paren_fallback_regexes = [] + without_paren_to_target = {} + + choices = doc["choices"] + for c in choices: + m = filter_ignores(c.strip()) + fallback_regexes.append(f"{re.escape(m)}") + choice_to_alpha[m] = f"({next_alpha})" + + without_paren_fallback_regexes.append(next_alpha) + without_paren_to_target[next_alpha] = f"({next_alpha})" + + next_alpha = chr(ord(next_alpha) + 1) + fallback_regex = re.compile("|".join(fallback_regexes)) + without_paren_fallback_regex = "|".join(without_paren_fallback_regexes) + without_paren_fallback_regex = re.compile( + rf":[\s]*({without_paren_fallback_regex})" + ) + + filtered = [] + for resp in r: + match = find_match(self.regex, resp) + if not match: + match = find_match( + fallback_regex, filter_ignores(resp), choice_to_alpha + ) + if not match: + match = find_match( + without_paren_fallback_regex, resp, without_paren_to_target + ) + if not match: + match = self.fallback + filtered.append(match) + filtered_resps.append(filtered) + + return filtered_resps diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20236ae8e25cbcc8cd43dd54157614518eea26d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_swa_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61ffc3f938f8cf13da5bac35bed1c6d2a9323acf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "For mathematical questions provided in Wolof language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_wol_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b52cfb72dfe0f89092718c46e6fd4360a5dd3646 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_zul.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "For mathematical questions provided in Zulu language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_zul_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e65e9298656895f4dab45111420da97559d62023 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ewe_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..400bf8887718fbe40d6701cb2121a3cd271c1360 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_kin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22599e98c83a4eae8b0c6b103396195af55fbbee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca19eb14ef29d16affc7efe255e3faa3ae4deb06 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_orm_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f8fc2ef28d888800670a9f3068f0d58d670d5ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sot_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0545cccda86167036dc321b11d29c1ca1ca2542 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_swa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee0d6fc9babb8f968cdcbbd460bdb83f14e14c06 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_wol_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f340a46529bd305f4bdc5ea73b7cd148ec6d7d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_xho_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ad7f0069cd8b63090b625ef103a37154356782c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yaml @@ -0,0 +1,33 @@ +tag: afrimgsm_tt_cot_tasks +dataset_path: masakhane/afrimgsm-translate-test +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a83764758c3e2540d3a097ed0e0f5ac987604a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ewe_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1022ae899b0e2413351e01aafef9de08b00688ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_hau_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd2a2528ef1c104f1714735f1a5b753c10966607 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ibo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70d4032301398b2124ff128ebb9ed1ba4eb0f0ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a774c189513e159a0d9cdf034cd0470cc25d8b84 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lug_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a298b504439fa6c7d8eab548ecf2a0b997eddc9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sot_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3de9a4c61b8a018007b13e16cd9028d31af92c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2e1ab61ec9af7af947ae656d2c7069230ee02c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_twi_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..185b406be03d84beef24b6c6fc453a4518d7a66f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_wol_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad059aead35b933cabaf763d549c59592f006fc7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yaml @@ -0,0 +1,33 @@ +tag: afrimgsm_tt_cot_tasks +dataset_path: masakhane/afrimgsm-translate-test +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: 'Give direct numerical answers for the question provided. \n\nQuestion: {{question}} \Step-by-Step Answer: ' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b627e57564d6839ec2ffde82c0a125e42a5c5b77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_amh_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..296ea98fc1dd70f50bb72ddb566d738d11d42f68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ace69b273060675411488fa639f261b2fb39f8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd25a1661f0720e68e2851701a1a9ed8f0131950 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lug_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..698c1474bd76edf931839d75b3111bafe8b0770c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_orm_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..354df6bfef4ea01a0b40b6361adb5d21470d395e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sna_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d86662980bf7078dde5636aba335bab9897619c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_swa_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0bb7d6661f0b78b5e417269c61f0e7fc028848f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yaml @@ -0,0 +1,33 @@ +tag: afrimgsm_tt_cot_tasks +dataset_path: masakhane/afrimgsm-translate-test +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: 'Solve the following math question \n\nQuestion: {{question}} \nStep-by-Step Answer: ' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08fbc9e15e1d963da67e5723f33b926701ca9503 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_zul_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d57247b8b512c60f5536cade1bbc7804083b2f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ewe_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf15ed077cd3ff2762deaf532d200e117fb6e9dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ibo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4deb09238913761e594ec4967a4dabd9d188b02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_orm_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ebac1993f84026c0823e52fb41c6574e816b4c7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sna_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b5ebb387b40e7a12a103a72ee43b3f711de9a7f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_translate +task: afrimmlu_translate_amh_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..192f3423797a807c355fc21bb4c1e9137cc31b72 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_translate +task: afrimmlu_translate_fra_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60d4d57f5bb6d793467316ac4fe2c2c97d055289 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_translate +task: afrimmlu_translate_sna_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..100bad5f4db1f86f9c37a871089ee2be3c5df926 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_hau diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1245d3da4e4d624c877e6014d5d3882d69f26889 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_kin diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7d94440efabe75519f5a6dbd04ff0bcc29e7b2b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_sna diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f90a2f5e50d6dcda86ef31b6b5c578c3639273ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_sot diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38e4ca57a413e0cd0b830f72adba13ffa281367b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_xho diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5d1ac19e19b2e855c957e75f1c778366dfbc7e55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "True", 1: "Neither", 2: "False"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb2dcd58252468e3fa9dcf8dbad7b39dc9d2983b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng +doc_to_choice: '{{[premise+", Right? Yes, "+hypothesis,premise+", Right? Also, "+hypothesis,premise+", + Right? No, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_eng diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecdc41c524eaf1429756643110a85b83009bb293 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ewe.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_choice: '{{[premise+", Esษ” gbe? ฦฬƒ, "+hypothesis,premise+", Esษ” gbe? Haฬƒ, "+hypothesis,premise+", + Esษ” gbe? Ao, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_ewe diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e6e32cc165ce73e99ae5480debe0183dbb2351a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_fra.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra +doc_to_choice: '{{[premise+", correct? Oui, "+hypothesis,premise+", correct? Aussi, + "+hypothesis,premise+", correct? Non, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_fra diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b243a5de37f970dc92f27112280332cd2c5256cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau +doc_to_choice: '{{[premise+", Daidai? Ee, "+hypothesis,premise+", Daidai? Haka kuma, + "+hypothesis,premise+", Daidai? A''a, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_hau diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..597ffb644c6e63a9bb4caccc41f45db2c2f29e68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_choice: '{{[premise+", Ziri ezi? ร‰รจ, "+hypothesis,premise+", Ziri ezi? แปŒzแปkwa, + "+hypothesis,premise+", Ziri ezi? Mba, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_ibo diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97b6d00ec8b4ed0d9e1c9aabe133cc0b70141dbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_lug.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lug +doc_to_choice: '{{[premise+", Kituufu? Yee, "+hypothesis,premise+", Kituufu? Nโ€™ekirala, + "+hypothesis,premise+", Kituufu? Nedda, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_lug diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9c25496da9cc81a3c82c8ae2a83621bf839e56a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_orm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm +doc_to_choice: '{{[premise+", Sirrii? Eeyyee, "+hypothesis,premise+", Sirrii? Akkasumas, + "+hypothesis,premise+", Sirrii? Lakki, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_orm diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be2b2617ccdec63258607be20a6db2d958f018b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sna.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna +doc_to_choice: '{{[premise+", Chokwadi? Hongu, "+hypothesis,premise+", Chokwadi? Uye, + "+hypothesis,premise+", Chokwadi? Kwete, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_sna diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..092961e0f8e39bb94a152aae00651b9ae49eebfb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sot.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sot +doc_to_choice: '{{[premise+", Nepile? E, "+hypothesis,premise+", Nepile? Hape, "+hypothesis,premise+", + Nepile? Tjhe, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_sot diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8b1e2afa2c1b267c803565caa9cc13dc8d8f506 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa +doc_to_choice: '{{[premise+", Sahihi? Ndiyo, "+hypothesis,premise+", Sahihi? Pia, + "+hypothesis,premise+", Sahihi? Hapana, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_swa diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d3141d63a84b5a93002bd416583eb444543c031 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_choice: '{{[premise+", Nifa? Aane, "+hypothesis,premise+", Nifa? Anaasษ›, "+hypothesis,premise+", + Nifa? Daabi, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_twi diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1239fa47086826050a23d493c3e7069327a0e516 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_choice: '{{[premise+", Dรซgg? Waaw, "+hypothesis,premise+", Dรซgg? Itam, "+hypothesis,premise+", + Dรซgg? Dรฉet, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_wol diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6f91f6e079d1138e374c7094bd76ef4743ec5b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_xho.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho +doc_to_choice: '{{[premise+", Ichanekile? Ewe, "+hypothesis,premise+", Ichanekile? + Kananjalo, "+hypothesis,premise+", Ichanekile? Hayi, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_xho diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5ec109bbd64b5cdbbd27329fa0bd7c67767cf5c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yaml @@ -0,0 +1,25 @@ +tag: + - afrixnli + - afrixnli_native_direct +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_text: "" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2648bf57bce8ffa5d28578094b477c6b8b166446 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_choice: '{{[premise+", ร’รณtแปฬ? Bแบนฬแบนฬ€ni, "+hypothesis,premise+", ร’รณtแปฬ? ร€ti pรฉ, + "+hypothesis,premise+", ร’รณtแปฬ? Rรกrรก, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_yor diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48261c60b28fa4c157b511153b609c840fea80e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_zul.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul +doc_to_choice: '{{[premise+", Kulungile? Yebo, "+hypothesis,premise+", Kulungile? + Futhi, "+hypothesis,premise+", Kulungile? Cha, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_zul diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/native-direct/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94fb2bdcb6f44646e6711dfaa38d7d0f66c767f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_translate_yaml +task: afrixnli_translate_amh diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55d5b470a2fdc47c73ac9ebeabbc6bdf388db0f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd5903357dbd029bbc5a3d88c47e75ab05b4da41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_translate_yaml +task: afrixnli_translate_fra diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddc7a375e03210ad02090a0279fe767e67d76c8e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_translate_yaml +task: afrixnli_translate_hau diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2487f15a4a75ede35ab29300b6764fabd325e139 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebae340f5bf3b21a5d72c1ed4f6bad6223834d27 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_translate_yaml +task: afrixnli_translate_kin diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ad2ea078f78d509c452850ec1fdeef2c1f96325 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_translate_yaml +task: afrixnli_translate_lin diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9ab91826d3f8b64370b061b58dcd5cd1b5d0da8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_translate_yaml +task: afrixnli_translate_lug diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..879228120a74794e894cad0b6d32ccb0b35ad473 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_translate_yaml +task: afrixnli_translate_orm diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64b5cb29c770a1380a69001edf0026f47a0509a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_translate_yaml +task: afrixnli_translate_sot diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea6307131bfe14bdf9951b929556b0e911bed25f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_translate_yaml +task: afrixnli_translate_swa diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cfd32e21ffd7208af52822be3bbeacd1676efab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_translate_yaml +task: afrixnli_translate_twi diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be1188e5cc7c941099bf25d9d7b71eba768dcf9a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_translate_yaml +task: afrixnli_translate_wol diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..428ff3bbd2dccc0f60bb3818860d8426f9f70739 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_translate_yaml +task: afrixnli_translate_xho diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f1df47cf06db0a549f279ee81ea5e8d8945a85e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yaml @@ -0,0 +1,32 @@ +tag: + - afrixnli + - afrixnli_translate +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither?\nAnswer:" +# True = entailment +# False = contradiction +# Neither = neutral +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "True" + - "Neither" + - "False" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f07a41a5b1b1f48ff85110aa3c6d1197a51f437 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_translate_yaml +task: afrixnli_translate_yor diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a57632bcafc9da38097eea7fbad89c14fbd12e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_translate_yaml +task: afrixnli_translate_zul diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5d1ac19e19b2e855c957e75f1c778366dfbc7e55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "True", 1: "Neither", 2: "False"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/afrixnli.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/afrixnli.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d85ccd128f752f7a1ab566aa28e90d5bbf545b66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/afrixnli.yaml @@ -0,0 +1,13 @@ +group: afrixnli-irokobench +task: + - afrixnli_tasks_prompt_1 + - afrixnli_tasks_prompt_2 + - afrixnli_tasks_prompt_3 + - afrixnli_tasks_prompt_4 + - afrixnli_tasks_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39f727b4ccb7c07eb0b2f6b8d2472764446767d4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_amh.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_amh_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..593c57a34ec0f01d3c03e447acda48cd1644231b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_eng.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6a10baae753575ebb228a6df34e4faf364efea1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ewe.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_ewe_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08b2b5243633f276487d8d5595382211870eedf9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_fra.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_fra_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe234b72694fdfde8474863d81265e851a350368 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_hau.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_hau_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d282e0e5f84433b77f8954acaa4e65c9ccbf5ba4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ibo.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_ibo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfdff6c8c64e6a91a869386f71b6f6024c0ac156 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_kin.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_kin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..410cb29f80366d78d0ec4fb9e240a9df0ec20372 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lin.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_lin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5665e37cce68d072cf8b64be5a787aed23fd70b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lug.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_lug_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12751c7f93de40a8bc91431de573be9f99868ab4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_orm.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_orm_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d00bbb6f9d146effdf5de3112db4c56f79002166 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sna.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_sna_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ae346aed8ff8e53b94252385443b54fe4364595 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sot.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_sot_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca6729bf80cad6e95027c7c0e994cd1da14d0d6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_swa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7dc85428dab2ed5da0cb6fa17b0a428088f346f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_twi.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_twi_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78ef254aeed1d393efca6390fa574aa41eac5f21 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_wol.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_wol_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb0a8527741f4b8f3ac1fb7c2741a6cf5e2c64ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_xho.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_xho_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yaml new file mode 100644 index 0000000000000000000000000000000000000000..81c9eeaa5af0740cc32122519f671c4d0425c080 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_1 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..473aea37a7b036d7ef219eca756482cd2bac754b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_yor_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa07a8c991d56a0cef3dc8453017649952715f8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_zul.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_zul_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_1/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbf916b25d43db2fdc476c27fe5f2e8e02c45625 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_yaml +task: afrixnli_amh_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfa8ebfe8815a58c1a043b328a22f762a739b9d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrixnli_yaml +task: afrixnli_eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..995ef3e65894548266a72f45417222e2760e30fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_yaml +task: afrixnli_ewe_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce72588c19901c04ee82479206f54816fa358915 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_yaml +task: afrixnli_fra_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..369ee58bedfd98fd95c63510c3a84eec10238df0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_yaml +task: afrixnli_hau_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e118c613ebf2d0388298fe6ba750923816ba4af6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_yaml +task: afrixnli_ibo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81f6d803d6762f5a6b86dae00ec0b26040a943ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_yaml +task: afrixnli_kin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d99c2eb57aedcce988f37415c414882d5bb4186 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_yaml +task: afrixnli_lin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31325539e1dd778da5e057436aeb3b60d7531a58 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_yaml +task: afrixnli_lug_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4ad555afafe2b99470d706e0eb46dc8256037fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_yaml +task: afrixnli_orm_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a780b0c428c822c08d5bb16dd909cb883da494a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_yaml +task: afrixnli_sna_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94e78880d31b48ce8dc4e562a9d9cc3643208535 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_yaml +task: afrixnli_sot_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8622e2833c24290007405dc90043f0c5b6ced7ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_yaml +task: afrixnli_swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4219b81ee8a1de24535cf2cd6eae4643e660d0de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_yaml +task: afrixnli_twi_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..546b17904959bd83c1f26617a9a13b79fc654a55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_yaml +task: afrixnli_wol_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..649c61df93eef6b6828d7da9c5a672bc5fce9611 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_yaml +task: afrixnli_xho_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfab642bf9175d3066879680618e95f097a609a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yaml @@ -0,0 +1,34 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_2 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither?\nAnswer:" +# True = entailment +# False = contradiction +# Neither = neutral +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "True" + - "Neither" + - "False" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53f23ace6bbb25c6457f7bd4e5b760b7ebb8b298 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_yaml +task: afrixnli_yor_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd89fe131e26f4b0d1dedb1b86aeac72f8f706d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_yaml +task: afrixnli_zul_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ff9f99c187a914fde7514c7c4caf49cb63c4186 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_amh.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Given the following premise and hypothesis in Amharic, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_amh_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a53aea6dbd9c3fedd9a812fc8f698b5f16d41bf3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_eng.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Given the following premise and hypothesis in English, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54b58ae6e972774be55be9988a20d6962a7e56ff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ewe.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Given the following premise and hypothesis in Ewe, identify if the premise\ + \ entails, contradicts, or is neutral towards the hypothesis. Please respond with\ + \ exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \n\ + Hypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ewe_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fedb519ec8421d77aedb24215de643544614bf70 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_fra.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Given the following premise and hypothesis in French, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_fra_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a9ebb95181426ab8a9138267a8400c37825e76e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Given the following premise and hypothesis in Hausa, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_hau_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b61f7678a3ab596bbda1b6039129e2e71b6bda6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Given the following premise and hypothesis in Igbo, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..697c439fa18a1ecd80e41fca14bc0836d956cfde --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Given the following premise and hypothesis in Lingala, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b5667c0720473f0ab7703b17777b5ced0459381 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lug.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Given the following premise and hypothesis in Luganda, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lug_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dabd96ef2e1cbf6df83432cc057382d40e448ff7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_swa.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Given the following premise and hypothesis in Swahili, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_swa_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d3158d45209fe96a7e0b7520d065c9108a0798b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_twi.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Given the following premise and hypothesis in Twi, identify if the premise\ + \ entails, contradicts, or is neutral towards the hypothesis. Please respond with\ + \ exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \n\ + Hypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_twi_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51fbdc79b0987386839599a3521bb2d564256e83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Given the following premise and hypothesis in Wolof, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_wol_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00ca9d17934256c4169ba0da95f7a604c60ac037 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_xho.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Given the following premise and hypothesis in isiXhosa, identify if\ + \ the premise entails, contradicts, or is neutral towards the hypothesis. Please\ + \ respond with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_xho_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yaml new file mode 100644 index 0000000000000000000000000000000000000000..04609ac3c424b323858858fdbffbd83ccec52b7e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_3 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d8b2f847f473ad2d1857e924495e98fffbc9edd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yor.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Given the following premise and hypothesis in Yoruba, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_yor_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63b05465144af310263939fea2b8335672dbb7ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_amh.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Amharic language.\nAnalyze the premise and hypothesis given in Amharic, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_amh_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..811a0fca1364f55e5ba3dfe37e7d9c99e7090e6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Hausa language.\nAnalyze the premise and hypothesis given in Hausa, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_hau_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73fdba2fba55cc9af3bb802e50562de8ceb9a97e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Igbo language.\nAnalyze the premise and hypothesis given in Igbo, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ibo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63062ac444bcd71951152416384ae5852510decb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Lingala language.\nAnalyze the premise and hypothesis given in Lingala, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lin_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba2a377b7fb03f6cd1546fe8f1b65549d2133d6c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_orm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Oromo language.\nAnalyze the premise and hypothesis given in Oromo, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_orm_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afce6e955b21eb514b4dfd024d7a8d115a3377ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sna.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the chiShona language.\nAnalyze the premise and hypothesis given in chiShona,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sna_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40c7cf8476813d29347fdd8e14785cb61a48c172 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sot.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Sesotho language.\nAnalyze the premise and hypothesis given in Sesotho, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sot_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9835314e7fc979010c80e6be80ebd616eb3abff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_twi.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Twi language.\nAnalyze the premise and hypothesis given in Twi, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_twi_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45f55e0e1441fe40f0a6fcecec0309d9c3013dc3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_xho.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the isiXhosa language.\nAnalyze the premise and hypothesis given in isiXhosa,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_xho_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe5de1a6dd271c23b14712b09ab070ec848b753b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_4 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63d4f60642c71a1d881af27291a73e04b4abca34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yor.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Yoruba language.\nAnalyze the premise and hypothesis given in Yoruba, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_yor_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70873a211527bec45adf7c689deef653eb3cfe07 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_amh.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_amh_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..675264a8dc0da305ec2d92ff16a8393fd9bd0729 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_eng_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bb558dabcb62d4e7c49c54c100986703fdc88ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_fra.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_fra_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..728ae1b805f2ac9f014200ad59c82b6e822ca884 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_hau_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3086b9b4f3c31121e089a8f0bc4c9e9ee4c1cc4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ibo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0250f29f300f3f0d312744b0c7a83bfbcc1bc55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lin_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..928b74ce4fce73f952ac71999b4dbfc83c9632cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lug.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lug_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac0ef3007edbcdcaf6a705202565ec1d842889a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sna.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sna_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21fcdde5b66733a9f488c12b207545241b7ee7e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sot.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sot_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d5824adcf3373864aa3ecf660952f667c648ea8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_swa_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b519ef71eec7ddf91fc2e247021779edfec29145 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_twi_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a865c8b166b19e458c3ff68138f74e48f8ce6b60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_wol_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yaml new file mode 100644 index 0000000000000000000000000000000000000000..13e2b6ef7244d2689d7c56146aa15328e792c2fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_5 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "true" + - "inconclusive" + - "false" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4510441b606a8dbc0e635a00c3a009c4f891bd23 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_yor_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6b9cb312b25a4c21bdd3d6a5e0a4e8e160451e4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "true", 1: "false", 2: "inconclusive"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e8974c99a1b5d8ebed9c9be29e3628ad7d41674 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_amh diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7936a4322a3948093eefc12364f47b25181b0227 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_eng diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..885e571b344e78cf0277dc5f3193dc6096386d40 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_hau diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a8428267e752a552c6bc67baaefd4a65f1bf47f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_ibo diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eed83c757faa913a2b220ced61eeb718c5da3c12 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_lin diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c3cc02445be42a6bdb4860b6300f59e5dbc622c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_lug diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad7660a03668f00c0bf5a46c1162d32f382831ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_sna diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..266605cb8c0deffca5020416e45d77a444b8f313 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_sot diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07a890927933a0dc665b98f7e56cbd620fa97b18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_swa diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d76fff819e09299df81564cc8217a2f34e20afbf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_twi diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..307b42fc58bf7782823473fe67a2343698c8ae9a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_xho diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yaml new file mode 100644 index 0000000000000000000000000000000000000000..3147dd0e24619dab8e927dc40ba54110b3e70c49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yaml @@ -0,0 +1,31 @@ +tag: + - afrixnli + - afrixnli_manual_direct +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2b9f99a05509897253504e57671be3df94adaf7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_yor diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2833840644b75044470a2dfb133d0afd43da105c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_zul diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d209484bdec5139ce18e3c84b9385cbe5549928 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_ewe diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a943963b9075e818074e98fcd3bf255502dd482a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_fra diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a343c952fe31f91f2332204df366a5434fd62f03 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_hau diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0057e6b0cbfbd7a7663829a05e0d44d60c301d3f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_ibo diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6e6023cf5ad47cede58060b973ee9aed9964bde --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_lin diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bc3a14d41eba99d4bb9f2b46fd44ec1526507cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_lug diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb9a494f4057783cca9a68eda9b4fb56e0b99948 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_orm diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6523987f10926fa2f2fd80417e86e494363f0fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_sna diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..319e909c84cf513aa9985a0a6cc44794f78a09b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_sot diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a34eb438e4d45bdedc68f893af2fb4374fc931a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_swa diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0535f1db84f43aaed989efcfbe9e1781480931b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_twi diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8541b73ec8e1c0c7417a5547cdfd170ed9bcf21b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_wol diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d1eebcb37f65f1ad44a098220f979aa840b4f57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_xho diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..089dc446943a5ca2c405e2ff71fcfdfa5cfb8b89 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yaml @@ -0,0 +1,29 @@ +tag: + - afrixnli + - afrixnli_manual_direct +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/afrixnli_tt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/afrixnli_tt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba507b39d0320bc6307062fc3158a2f1d9212c84 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/afrixnli_tt.yaml @@ -0,0 +1,9 @@ +group: afrixnli_tt-irokobench +task: + - afrixnli_tt_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92ef8df7270120b2ead5d3ece0d9cffc2bfc1741 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_amh.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_amh_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa32dd72a6bd7412eae5fb94ccb3d5af06402a1c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ewe.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77f22faf2789d56de00b4a226832e2cb3d401362 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_hau.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_hau_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7ac8793976d7722375f01f43f587b74e9654ec2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ibo.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a52861402a8c3d65ccbe025fa62807d86e89b14 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_kin.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_kin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb2a667e864afb856ec85ecd6b300378b44f8050 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lin.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_lin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14b20a1c35ae63adddddbc4a0b8d4e1fba2c90b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_orm.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_orm_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97cf3cba569f25219a7cdccb2be7a1c6effae0c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sot.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_sot_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..877787a88c0e8ee0c2d7d87d97aeff2d91e1330f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_wol.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_wol_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c0ec7c91ff7d81a500bb1d335fe06406a3c94e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_yor_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78753d1fe3ef143c547780041e70cc03d20289f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_zul.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_zul_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f17a4ecf0a3fe044c04fff53ef61f5946bf744b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_translate_yaml +task: afrixnli_translate_fra_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5667b3d0624c569bf9e392a67780a80fdf5aee3b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a74950cc34121f63d3dd944a310b7656bc2ff894 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_translate_yaml +task: afrixnli_translate_kin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63ff988c3aea7178e06d85ca20fea40c87e9dbcc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_translate_yaml +task: afrixnli_translate_lug_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..926f91321cfcade46ca52492cba83daa348bc746 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_translate_yaml +task: afrixnli_translate_swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ae3c21e2100b4b07428f67c23070d51d32761b0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Given the following premise and hypothesis in Hausa, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_hau_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28696f337841ae738354ae11bf537f07743c49f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Given the following premise and hypothesis in Igbo, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebc775dd705e572f68835e194b6c3c8d745f6e1b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ewe.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Ewe language.\nAnalyze the premise and hypothesis given in Ewe, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/README.md b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/README.md new file mode 100644 index 0000000000000000000000000000000000000000..10d46a44e2098e7e8aaadf57dbdfe5eb52156d7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/README.md @@ -0,0 +1,41 @@ +# + +## Paper +Title: `The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants` + +Paper Link: https://aclanthology.org/2023.emnlp-main.862/ + +## Abstract +>Belebele is a multiple-choice machine reading comprehension (MRC) dataset spanning 122 language variants. This dataset enables the evaluation of mono- and multi-lingual models in high-, medium-, and low-resource languages. Each question has four multiple-choice answers and is linked to a short passage from the FLORES-200 dataset. The human annotation procedure was carefully curated to create questions that discriminate between different levels of generalizable language comprehension and is reinforced by extensive quality checks. While all questions directly relate to the passage, the English dataset on its own proves difficult enough to challenge state-of-the-art language models. Being fully parallel, this dataset enables direct comparison of model performance across all languages. Belebele opens up new avenues for evaluating and analyzing the multilingual abilities of language models and NLP systems. + +HomePage: https://github.com/facebookresearch/belebele + +### Citation + +``` +@inproceedings{bandarkar-etal-2024-belebele, + title = "The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants", + author = "Bandarkar, Lucas and + Liang, Davis and + Muller, Benjamin and + Artetxe, Mikel and + Shukla, Satya Narayan and + Husa, Donald and + Goyal, Naman and + Krishnan, Abhinandan and + Zettlemoyer, Luke and + Khabsa, Madian", + editor = "Ku, Lun-Wei and + Martins, Andre and + Srikumar, Vivek", + booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = aug, + year = "2024", + address = "Bangkok, Thailand", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.acl-long.44/", + doi = "10.18653/v1/2024.acl-long.44", + pages = "749--775", + abstract = "We present Belebele, a multiple-choice machine reading comprehension (MRC) dataset spanning 122 language variants. Significantly expanding the language coverage of natural language understanding (NLU) benchmarks, this dataset enables the evaluation of text models in high-, medium-, and low-resource languages. Each question is based on a short passage from the FLORES-200 dataset and has four multiple-choice answers. The questions were carefully curated to discriminate between models with different levels of general language comprehension. The English dataset on its own proves difficult enough to challenge state-of-the-art language models. Being fully parallel, this dataset enables direct comparison of model performance across all languages. We use this dataset to evaluate the capabilities of multilingual masked language models (MLMs) and large language models (LLMs). We present extensive results and findings, notably that despite significant cross-lingual transfer in English-centric LLMs, much smaller MLMs pretrained on balanced multilingual data still understand far more languages. Overall, Belebele opens up new avenues for evaluating and analyzing the multilingual capabilities of NLP systems." +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele new file mode 100644 index 0000000000000000000000000000000000000000..51553e0e077d968e1fca29e27783b225ccaf7323 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_1 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b2ef7a14b8b1f566b1a55b6e8e02e12b327becb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lug.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_lug_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b667c1d66cff764f6d57cfa6ff698796b608e446 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_luo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_luo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c220c7738f369cc8255efc2b96c4d9654a5c33f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_nya.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_nya_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eec0b1e19fe1bdd53a5add1d16b6b9a89e12a213 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sna.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_sna_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10cde5be846773e764b762157bc528a5acf3fc1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sot.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_sot_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c4ae7b786261ef4e7b48bc35c077207a232bb31 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_swa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b62e848daa5776384c780acce0b270cf88607c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tir.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_tir_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tsn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tsn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..147a1c9857be7f9557759765a7a6d5d490564bf7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tsn.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_tsn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1aed1e5ac7954377f0e594e5e2f3b8260d140c62 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_wol.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_wol_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..257396921dbe6be6b5954170511093b382e9ba9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_zul.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_zul_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele new file mode 100644 index 0000000000000000000000000000000000000000..75f673a425116056c7516d0b7e8f54844f0c9716 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_2 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7c7180959511b34bc8582976aa14f1b4327ad1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_afr.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_afr_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a975b485cd567a5f574add79bc5651e58d533219 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_arz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_arz_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..510f1fbb0d2070f3c547fbe76f3c5a69e4ba31f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_eng.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bf02ff0fa62983d5e4c9c8e63129b1e40da333a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fuv.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_fuv_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7af70e03fa6ab6f868205f2ad2b5abc4727a0dfb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_hau.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_hau_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92d895803b9927d2a8288a7b9ff4493f23476c77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_ibo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f1dcf9117db0c0ef3065975c7bce8abab626ae1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kea.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_kea_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe97881b310795720928c4c43241c076297903c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lug.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_lug_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57e71ac9c186e22ac235f85781ffb6400e41eb11 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_plt.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_plt_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9be02a8e500875dfc2f5b20c1d0f6d3767dce93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_por.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_por_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a5ad43a21f7a2bd32933b47e9d5aa10a623cff3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sna.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_sna_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d551d1d5c501ddc309b37b8688898359e44ecc3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_som.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_som_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18780c9017658355fc160320784f848867bf03cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sot.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_sot_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6bd0a69ac685b18d7d03eaa2deeb120913904ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ssw.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_ssw_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bdfd132cd8ac856e157d2dd502f7f030e95c0c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fba28cba714edf37a5169da7ba427543b1dd8e2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tir.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_tir_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tsn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tsn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..337e08ceab543cad55a4ca0206bbf70590422642 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tsn.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_tsn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a0e24e4a4118e26150288d0f6842ec2baeec12a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tso.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_tso_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..439148106cf268746b7e41695a752a9ba8422a6c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_wol.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_wol_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2077614892f77249687bf7b8889a211212325e83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_xho.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_xho_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9684f05db11c39ea1e000f1b31ce04b3693cc01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_yor_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c81180e17c7f71799e36536a4abcdb707ef7652d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_zul.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_zul_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele new file mode 100644 index 0000000000000000000000000000000000000000..a27ea5fb3a06cf7d949c6bc464c721a697e8d981 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_3 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e6f2fd77b913715ca3d24c8cf209a46dca1398b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_amh.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_amh_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..203bf1c9239b45803f91c601047567aaa01c8ed9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ary.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_ary_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b5d3415a3008831a38f2f6e3abb09eb79ba072e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_bam.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_bam_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ceb5270ec60454b3322a5058307bfce92caa3f8d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_eng.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..affc5d12fd30c95380804fdeb538a50fd6bcf582 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fra.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_fra_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ff7bfdad8e19ae4d4f1bc3a94e6158721243b88 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fuv.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_fuv_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c067e7c1972edd8109312d4a313fa81d62f3effc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_gaz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_gaz_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5eaacad2b56cae67a575640a79df06e87624156 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c24b2ae7a28c976c7d3980ad86b61148aa0d1635 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kea.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_kea_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae0a821fc67dd16965c9e46dc0f8724c508dc7bb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_kin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93e1a5b5438d97012207f12d9bc88a639252644f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_lin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..724947d41b746ee2507c1340003dbda425c89ddb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lug.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_lug_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21b4b8f730f3f24abaf93b0a71badee8f6c16819 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_luo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_luo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db045f723e91ed1e04317ad29ff7c7ee5bafa274 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_nya.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_nya_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..946e417946c1c066f2c552d42e266fac048bf795 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_plt.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_plt_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72ca651b8c251e1dfd304c9e6312b8230c2e2b1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_por.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_por_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f5d810ac1ec1bca42f68bf73da7a9ba10a2315a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sna.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_sna_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d3a7c4e46094d0eaa70553ecf1c6ff95b557379 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_som.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_som_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3db32d81ad3f709a2889da57042411ce00c8f294 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sot.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_sot_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..888ecf8423b80a3ac57980a91658c0f0c6a83254 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ssw.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_ssw_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec8127aae7cb28ddcfae818b80c6d58bb71c70fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_swa_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab3545445999b415cc5a81864672db07f4cf548d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tir.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_tir_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tsn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tsn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..019a95fe9a49e4c3398d01d84af7fcfc72cf1214 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tsn.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_tsn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcc97c4f09bc6abc5129d37ef2dabbbbcd6b71b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tso.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_tso_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20af7b3c57d384f4f63461f47240d9e4cadb91e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_wol.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_wol_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a205da905597abd079e2a8131b37e63c2d1a13f6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_xho.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_xho_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cdcbb8c244275b55670b542247d8db79c6b3bf54 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_yor_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da1ef4239b68c83c5ceaa1988a85329275db97fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_zul.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_zul_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele new file mode 100644 index 0000000000000000000000000000000000000000..cc28101b1072e9ef48202c143592df0ff2f8286b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_4 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02eb0683fb6fdd1a603f3c7d864f58a3ada9c458 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_amh.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_amh_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c7899d23cf3bb34c56bb5c1b866397deb6d96be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ary.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_ary_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5acc3222b7bae84c471027e21afb2b6930ac7848 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_arz.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_arz_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..466dddff4989fe2ba2e26959fd3f47d40ebba425 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_bam.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_bam_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21dfa3ea83d46444712c98657b68cd2d4416ba1c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_eng.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_eng_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7fea6f1ba3e5792d2ebe755111c5d924ad07999 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fra.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_fra_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77fa7798b1b079a65146976e0ecbf68ce94504cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fuv.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_fuv_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9e54eb9b9a74a929ecc7fff7313bb2783f7a8d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_gaz.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_gaz_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45dbfc5730a25b7945cc06efb258da867f86c690 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_hau.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_hau_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb58d8a079a7c0fd1fbeb30976361190d91f067f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ibo.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_ibo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8ce83009df730072f0d40dd8090d103c2825a9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kea.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_kea_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..028de73a190394f6c7c7de22f24060294cf1da3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kin.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_kin_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95cad4e2f44fc85e0dd9276c0024de0f1d2be617 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lin.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_lin_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce5ec04ac9e580585168a7a26b7f180ccba450f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_luo.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_luo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3869a67959fbff48a93174161750246ee0ccc510 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sna.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_sna_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec30bccc32adc5a5d7a7d6789a4dfb3941d3cc4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sot.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_sot_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..510e7b8f2a6d7528fc8162f00bdbaa7b6a89ac4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ssw.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_ssw_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tsn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tsn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8f0a28d86b665bc913b1760dc378e8e4bf4146d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tsn.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_tsn_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3510a4d82e5975dde9272cd83e856780d7a3766 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_xho.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_xho_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7472e5213b9a5a016818e89eaf901611423131b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_zul.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_zul_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01a724719757a2655800615e982a1ff1272dc438 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_afr.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_afr_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f707d7c38153cd8304f7f02e331dc00858cb59c3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_amh.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_amh_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..704c41a5ec8a68081ac33ea39b72f962e75e130f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_bam.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_bam_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a0a53114456c6223050d8a444052e2a15b3e2aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_hau.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_hau_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5a8e29bcd708791b528111da1b3301586825561 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ibo.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_ibo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bd9a07b8853165e8b5022a2d110cd450f2708ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kin.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_kin_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f81859aae5913c79df2bb245fe48016dba0920a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_luo.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_luo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c957760af620a7bf10726fb9611baa5758d1a03d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_nya.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_nya_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13b4e63948d0bc6ba9c886490a8585d2339bd357 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_por.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_por_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/README.md b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ccf433a9f884576ef412148ea67e1a07c86bea30 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/README.md @@ -0,0 +1,31 @@ +# + +## Paper +Title: `The FLORES-200 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation` + +Paper Link: https://arxiv.org/abs/2207.04672 + +HomePage: https://huggingface.co/datasets/facebook/flores + +### Citation + +``` +@article{nllb2022, + author = {NLLB Team, Marta R. Costa-jussร , James Cross, Onur ร‡elebi, Maha Elbayad, Kenneth Heafield, Kevin Heffernan, Elahe Kalbassi, Janice Lam, Daniel Licht, Jean Maillard, Anna Sun, Skyler Wang, Guillaume Wenzek, Al Youngblood, Bapi Akula, Loic Barrault, Gabriel Mejia Gonzalez, Prangthip Hansanti, John Hoffman, Semarley Jarrett, Kaushik Ram Sadagopan, Dirk Rowe, Shannon Spruit, Chau Tran, Pierre Andrews, Necip Fazil Ayan, Shruti Bhosale, Sergey Edunov, Angela Fan, Cynthia Gao, Vedanuj Goswami, Francisco Guzmรกn, Philipp Koehn, Alexandre Mourachko, Christophe Ropers, Safiyyah Saleem, Holger Schwenk, Jeff Wang}, + title = {No Language Left Behind: Scaling Human-Centered Machine Translation}, + year = {2022} +} + +@inproceedings{, + title={The FLORES-101 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation}, + author={Goyal, Naman and Gao, Cynthia and Chaudhary, Vishrav and Chen, Peng-Jen and Wenzek, Guillaume and Ju, Da and Krishnan, Sanjana and Ranzato, Marc'Aurelio and Guzm\'{a}n, Francisco and Fan, Angela}, + year={2021} +} + +@inproceedings{, + title={Two New Evaluation Datasets for Low-Resource Machine Translation: Nepali-English and Sinhala-English}, + author={Guzm\'{a}n, Francisco and Chen, Peng-Jen and Ott, Myle and Pino, Juan and Lample, Guillaume and Koehn, Philipp and Chaudhary, Vishrav and Ranzato, Marc'Aurelio}, + journal={arXiv preprint arXiv:1902.01382}, + year={2019} +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/gen_utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/gen_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..37e22e13d6b024976e9198df78dfa7ae81845e8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/gen_utils.py @@ -0,0 +1,202 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang, lang_dict): + language_column_name = f"sentence_{lang}" + prompt_map = { + "prompt_1": f"{lang_dict[lang]}: {{{{{language_column_name}}}}} \nEnglish: ", + "prompt_1_reverse": f"English: {{{{sentence_eng_Latn}}}} \n{lang_dict[lang]}: ", + "prompt_2": f"You are a translation expert. Translate the following {lang_dict[lang]} sentences to English \n" + f"{lang_dict[lang]}: {{{{{language_column_name}}}}}\nEnglish: ", + "prompt_2_reverse": f"You are a translation expert. Translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish: {{sentence_eng_Latn}} " + f"\n{lang_dict[lang]}: ", + "prompt_3": f"As a {lang_dict[lang]} and English linguist, translate the following {lang_dict[lang]} sentences " + f"to English \n{lang_dict[lang]}: {{{{{language_column_name}}}}}\nEnglish: ", + "prompt_3_reverse": f"As a {lang_dict[lang]} and English linguist, translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish: {{sentence_eng_Latn}} " + f"\n{lang_dict[lang]}: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse: bool) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "ace_Latn": "Acehnese (Latin script)", + "ace_Arab": "Acehnese (Arabic script)", + "acq_Arab": "Taโ€™izzi-Adeni Arabic", + "aeb_Arab": "Tunisian Arabic", + "afr_Latn": "Afrikaans", + "aka_Latn": "Akan", + "amh_Ethi": "Amharic", + "ary_Arab": "Moroccan Arabic", + "arz_Arab": "Egyptian Arabic", + "bam_Latn": "Bambara", + "ban_Latn": "Balinese", + "bem_Latn": "Bemba", + "cjk_Latn": "Chokwe", + "dik_Latn": "Southwestern Dinka", + "dyu_Latn": "Dyula", + "ewe_Latn": "Ewe", + "fon_Latn": "Fon", + "fra_Latn": "French", + "fuv_Latn": "Nigerian Fulfulde", + "hau_Latn": "Hausa", + "ibo_Latn": "Igbo", + "kab_Latn": "Kabyle", + "kam_Latn": "Kamba", + "knc_Arab": "Central Kanuri (Arabic script)", + "knc_Latn": "Central Kanuri (Latin script)", + "kbp_Latn": "Kabiyรจ", + "kea_Latn": "Kabuverdianu", + "kik_Latn": "Kikuyu", + "kin_Latn": "Kinyarwanda", + "kmb_Latn": "Kimbundu", + "kon_Latn": "Kikongo", + "lin_Latn": "Lingala", + "lua_Latn": "Luba-Kasai", + "lug_Latn": "Luganda", + "luo_Latn": "Luo", + "plt_Latn": "Plateau Malagasy", + "mos_Latn": "Mossi", + "nso_Latn": "Northern Sotho", + "nus_Latn": "Nuer", + "nya_Latn": "Nyanja", + "gaz_Latn": "Oromo", + "run_Latn": "Rundi", + "sag_Latn": "Sango", + "sna_Latn": "Shona", + "som_Latn": "Somali", + "sot_Latn": "Southern Sotho", + "ssw_Latn": "Swati", + "sun_Latn": "Sundanese", + "swh_Latn": "Swahili", + "tir_Ethi": "Tigrinya", + "taq_Latn": "Tamasheq", + "taq_Tfng": "Tamasheq (Tifinagh script)", + "tsn_Latn": "Setswana", + "tso_Latn": "Tsonga", + "tum_Latn": "Tumbuka", + "twi_Latn": "Twi", + "tzm_Tfng": "Central Atlas Tamazight", + "umb_Latn": "Umbundu", + "wol_Latn": "Wolof", + "xho_Latn": "Xhosa", + "yor_Latn": "Yoruba", + "zul_Latn": "Zulu", + } + + for lang in languages.keys(): + try: + if not reverse: + file_name = f"flores_{lang}-eng_Latn.yaml" + task_name = f"flores_{lang}-eng_Latn_{mode}" + yaml_template = "flores" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": f"{lang}-eng_Latn", + "doc_to_target": "sentence_eng_Latn", + "doc_to_text": prompt_func(mode, lang, languages), + } + os.makedirs(f"{output_dir}/{mode}/african-english", exist_ok=True) + with open( + f"{output_dir}/{mode}/african-english/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + else: + file_name = f"flores_eng_Latn-{lang}.yaml" + task_name = f"flores_eng_Latn-{lang}_{mode}" + yaml_template = "flores" + # mode_reverse = f"{mode}_reverse" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": f"eng_Latn-{lang}", + "doc_to_target": f"sentence_{lang}", + "doc_to_text": prompt_func(f"{mode}_reverse", lang, languages), + } + os.makedirs(f"{output_dir}/{mode}/english-african", exist_ok=True) + with open( + f"{output_dir}/{mode}/english-african/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3"], + help="Prompt number", + ) + parser.add_argument( + "--reverse", + default=True, + choices=[True, False], + help="Reverse the translation direction", + ) + args = parser.parse_args() + + gen_lang_yamls( + output_dir=args.output_dir, + overwrite=args.overwrite, + mode=args.mode, + reverse=args.reverse, + ) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores new file mode 100644 index 0000000000000000000000000000000000000000..c25cf195cd032014435335eadf13e102f47598f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_afr-eng +- flores_afr-eng_prompt_1 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_acq_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_acq_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3634e7a66c7b11be9a0450f6f5ab953707897eb1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_acq_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: acq_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Taโ€™izzi-Adeni Arabic: {{sentence_acq_Arab}} \nEnglish: " +include: flores +task: flores_acq_Arab-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_afr_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_afr_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ac14a0c04b362f925e578a30a9eb615a6bc1fed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_afr_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: afr_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Afrikaans: {{sentence_afr_Latn}} \nEnglish: " +include: flores +task: flores_afr_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_arz_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_arz_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72552bab1ce9b04feb4619a28e4a424b1bdb99d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_arz_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: arz_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Egyptian Arabic: {{sentence_arz_Arab}} \nEnglish: " +include: flores +task: flores_arz_Arab-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/utils.py b/lm-evaluation-harness/lm_eval/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..bca19866ce7ac86c8c34bfac3f1f07a2c2345565 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/utils.py @@ -0,0 +1,552 @@ +import collections +import fnmatch +import functools +import hashlib +import importlib.util +import inspect +import json +import logging +import os +import re +from dataclasses import asdict, is_dataclass +from itertools import islice +from pathlib import Path +from typing import Any, Callable, Generator, List, Optional, Tuple + +import numpy as np +import yaml +from jinja2 import BaseLoader, Environment, StrictUndefined + + +SPACING = " " * 47 + +HIGHER_IS_BETTER_SYMBOLS = { + True: "โ†‘", + False: "โ†“", +} + + +def setup_logging(verbosity=logging.INFO): + # Configure the root logger + class CustomFormatter(logging.Formatter): + def format(self, record): + if record.name.startswith("lm_eval."): + record.name = record.name[len("lm_eval.") :] + return super().format(record) + + formatter = CustomFormatter( + "%(asctime)s %(levelname)-8s [%(name)s:%(lineno)d] %(message)s", + datefmt="%Y-%m-%d:%H:%M:%S", + ) + + log_level = os.environ.get("LOGLEVEL", verbosity) or verbosity + + level_map = { + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + "CRITICAL": logging.CRITICAL, + } + + log_level = level_map.get(str(log_level).upper(), logging.INFO) + + if not logging.root.handlers: + handler = logging.StreamHandler() + handler.setFormatter(formatter) + + root_logger = logging.getLogger() + root_logger.addHandler(handler) + root_logger.setLevel(log_level) + + if log_level == logging.DEBUG: + third_party_loggers = ["urllib3", "filelock", "fsspec"] + for logger_name in third_party_loggers: + logging.getLogger(logger_name).setLevel(logging.INFO) + else: + logging.getLogger().setLevel(log_level) + + +def hash_string(string: str) -> str: + return hashlib.sha256(string.encode("utf-8")).hexdigest() + + +def escaped_split(text, sep_char, maxsplit=-1): + """Split text into a list on occurrences of the given separation + character `sep_char`. The separation character may be escaped by a + backslash to avoid splitting at that location. + + The separation character must be a string of size 1. + + If `maxsplit` is given, at most `maxsplit` splits are done (thus, + the list will have at most `maxsplit + 1` elements). If `maxsplit` + is not specified or less than 0, then there is no limit on the + number of splits (all possible splits are made). + """ + assert len(sep_char) == 1, ( + "separation string must be a single character for escaped splitting" + ) + + if maxsplit == 0: + return text + maxsplit = max(0, maxsplit) + + return re.split(r"(? dict: + """ + Parses something like + args1=val1,arg2=val2 + Into a dictionary + """ + if args_string is None: + return {} + args_string = args_string.strip() + if not args_string: + return {} + arg_list = [arg for arg in args_string.split(",") if arg] + args_dict = { + kv[0]: handle_arg_string("=".join(kv[1:])) + for kv in [arg.split("=") for arg in arg_list] + } + return args_dict + + +def join_iters(iters): + for iter in iters: + yield from iter + + +def group(arr, fn): + res = collections.defaultdict(list) + + for ob in arr: + res[fn(ob)].append(ob) + + return list(res.values()) + + +# Returns a list containing all values of the source_list that +# match at least one of the patterns +def pattern_match(patterns, source_list): + if isinstance(patterns, str): + patterns = [patterns] + + task_names = set() + for pattern in patterns: + for matching in fnmatch.filter(source_list, pattern): + task_names.add(matching) + return sorted(list(task_names)) + + +def softmax(x) -> np.ndarray: + """Compute softmax values for each sets of scores in x.""" + e_x = np.exp(x - np.max(x)) + return e_x / e_x.sum() + + +def general_detokenize(string) -> str: + string = string.replace(" n't", "n't") + string = string.replace(" )", ")") + string = string.replace("( ", "(") + string = string.replace('" ', '"') + string = string.replace(' "', '"') + string = re.sub(r" (['.,])", r"\1", string) + return string + + +def get_file_task_name(filename: str) -> str: + """ + Given the sample results filenames, extracts and returns the task name. + """ + return filename[filename.find("_") + 1 : filename.rfind("_")] + + +def get_file_datetime(filename: str) -> str: + """ + Given the results and sample results filenames, extracts and returns the datetime. + """ + return filename[filename.rfind("_") + 1 :].replace(".jsonl", "") + + +def sanitize_model_name(model_name: str) -> str: + """ + Given the model name, returns a sanitized version of it. + """ + return re.sub(r"[\"<>:/\|\\?\*\[\]]+", "__", model_name) + + +def sanitize_task_name(task_name: str) -> str: + """ + Given the task name, returns a sanitized version of it. + """ + return re.sub(r"\W", "_", task_name) + + +def get_latest_filename(filenames: List[str]) -> str: + """ + Given a list of filenames, returns the filename with the latest datetime. + """ + return max(filenames, key=lambda f: get_file_datetime(f)) + + +def get_results_filenames(filenames: List[str]) -> List[str]: + """ + Extracts filenames that correspond to aggregated results. + """ + return [f for f in filenames if "/results_" in f and ".json" in f] + + +def get_sample_results_filenames(filenames: List[str]) -> List[str]: + """ + Extracts filenames that correspond to sample results. + """ + return [f for f in filenames if "/samples_" in f and ".json" in f] + + +def get_rolling_token_windows( + token_list: List[int], prefix_token: int, max_seq_len: int, context_len: int +) -> Generator[Tuple[List[int], List[int]], None, None]: + """ + - context_len allows for a rolling window context, allowing each prediction window to potentially + condition on some context + + :param token_list: list + List of tokens to be PREDICTED + :param max_seq_len: int + max_seq_len of model (or max_seq_len we want to use) + :param context_len: int + Amount of desired token context for prediction. Needs to be at least 1. + :param prefix_token: token + Dummy token like so the first token has something to condition on + :return: generator + Generator of tuples + (input_tokens, pred_tokens) + Note: Score only the last len(pred_tokens) logits of the LM + """ + assert 1 <= context_len <= max_seq_len + if not token_list: + return + # +1 offset, going from input->preds + pred_len = max_seq_len - context_len + 1 + predicted = 0 + + # Special handling for first window: predict all tokens + first_seq_len = min(max_seq_len, len(token_list)) + yield [prefix_token] + token_list[: first_seq_len - 1], token_list[:first_seq_len] + predicted += first_seq_len + + while predicted < len(token_list): + window_pred_len = min(len(token_list) - predicted, pred_len) + window_end = predicted + window_pred_len + + yield ( + token_list[window_end - max_seq_len - 1 : window_end - 1], + token_list[window_end - window_pred_len : window_end], + ) + predicted += window_pred_len + + +def make_disjoint_window( + pair: Tuple[List[int], List[int]], +) -> Tuple[List[int], List[int]]: + """Takes output from get_rolling_token_windows and makes the context not overlap with the continuation""" + a, b = pair + return a[: len(a) - (len(b) - 1)], b + + +class EnhancedJSONEncoder(json.JSONEncoder): + """ + Provides a proper json encoding for the loggers and trackers json dumps. + Notably manages the json encoding of dataclasses. + """ + + def default(self, o): + if is_dataclass(o): + return asdict(o) + return super().default(o) + + +class Reorderer: + def __init__(self, arr: List[Any], fn: Callable) -> None: + """Reorder an array according to some function + + Args: + arr (List[Any]): The initial array + fn (Callable[[Any], Any]): A function to determine the priority of elements + """ + self.size = len(arr) + arr = list(enumerate(arr)) + arr = group(arr, lambda x: fn(x[1])) + # arr = [([y[0] for y in x], x[0][1]) for x in arr] + # TODO: overhaul reorderer. It currently grouped requests by content but we don't want this + arr = [([y[0]], x[0][1]) for x in arr for y in x] + arr.sort(key=lambda x: fn(x[1])) + + self.arr = arr + + def get_reordered(self): + """Gets the reordered array + + Returns: + List[Any]: The reordered array + """ + return [x[1] for x in self.arr] + + def get_original(self, newarr): + """Restores the original order of a new array based on the old array's order + + Args: + newarr (List[Any]): The array to be restored + + Returns: + List[Any]: The array restored to the original order + """ + res = [None] * self.size + cov = [False] * self.size + + for (inds, _), v in zip(self.arr, newarr): + for ind in inds: + res[ind] = v + cov[ind] = True + + assert all(cov) + + return res + + +def make_table(result_dict, column: str = "results", sort_results: bool = False): + """Generate table of results.""" + from pytablewriter import LatexTableWriter, MarkdownTableWriter + + if column == "results": + column_name = "Tasks" + elif column == "groups": + column_name = "Groups" + + all_headers = [ + column_name, + "Version", + "Filter", + "n-shot", + "Metric", + "", + "Value", + "", + "Stderr", + ] + + md_writer = MarkdownTableWriter() + latex_writer = LatexTableWriter() + md_writer.headers = all_headers + latex_writer.headers = all_headers + + values = [] + + keys = result_dict[column].keys() + if sort_results: + # sort entries alphabetically by task or group name. + # NOTE: we default here to false, because order matters for multi-level table printing a la mmlu. + # sorting here would mess that up + keys = sorted(keys) + for k in keys: + dic = result_dict[column][k] + version = result_dict["versions"].get(k, " N/A") + n = str(result_dict.get("n-shot", " ").get(k, " ")) + higher_is_better = result_dict.get("higher_is_better", {}).get(k, {}) + + if "alias" in dic: + k = dic.pop("alias") + + metric_items = dic.items() + metric_items = sorted(metric_items) + + for (mf), v in metric_items: + m, _, f = mf.partition(",") + if m.endswith("_stderr"): + continue + + hib = HIGHER_IS_BETTER_SYMBOLS.get(higher_is_better.get(m), "") + + v = "%.4f" % v if isinstance(v, float) else v + + if m + "_stderr" + "," + f in dic: + se = dic[m + "_stderr" + "," + f] + se = " N/A" if se == "N/A" else "%.4f" % se + values.append([k, version, f, n, m, hib, v, "ยฑ", se]) + else: + values.append([k, version, f, n, m, hib, v, "", ""]) + k = "" + version = "" + md_writer.value_matrix = values + latex_writer.value_matrix = values + + # todo: make latex table look good + # print(latex_writer.dumps()) + + return md_writer.dumps() + + +def positional_deprecated(fn): + """ + A decorator to nudge users into passing only keyword args (`kwargs`) to the + wrapped function, `fn`. + """ + + @functools.wraps(fn) + def _wrapper(*args, **kwargs): + if len(args) != 1 if inspect.ismethod(fn) else 0: + print( + f"WARNING: using {fn.__name__} with positional arguments is " + "deprecated and will be disallowed in a future version of " + "lm-evaluation-harness!" + ) + return fn(*args, **kwargs) + + return _wrapper + + +def ignore_constructor(loader, node): + return node + + +def import_function(loader: yaml.Loader, node, yaml_path: Path): + function_name = loader.construct_scalar(node) + + *module_name, function_name = function_name.split(".") + if isinstance(module_name, list): + module_name = ".".join(module_name) + module_path = yaml_path.parent / f"{module_name}.py" + + spec = importlib.util.spec_from_file_location(module_name, module_path.as_posix()) + + if spec is None: + raise ImportError(f"Could not import module {module_name} from {module_path}.") + module = importlib.util.module_from_spec(spec) + + if spec.loader is None: + raise ImportError(f"Module loader is None, {module_name} from {module_path}.") + spec.loader.exec_module(module) + + function = getattr(module, function_name) + return function + + +def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None, mode="full"): + if mode == "simple": + constructor_fn = ignore_constructor + elif mode == "full": + if yaml_path is None: + raise ValueError("yaml_path must be provided if mode is 'full'.") + # Attach yaml_path to the import function so that it can be used later + constructor_fn = functools.partial(import_function, yaml_path=Path(yaml_path)) + + loader = yaml.CLoader if yaml.__with_libyaml__ else yaml.FullLoader + # Add the import_function constructor to the YAML loader + yaml.add_constructor("!function", constructor_fn, Loader=loader) + if yaml_config is None: + with open(yaml_path, "rb") as file: + yaml_config = yaml.load(file, Loader=loader) + + if yaml_dir is None: + yaml_dir = os.path.dirname(yaml_path) + + assert yaml_dir is not None + + if "include" in yaml_config: + include_path = yaml_config["include"] + del yaml_config["include"] + + if isinstance(include_path, str): + include_path = [include_path] + + # Load from the last one first + include_path.reverse() + final_yaml_config = {} + for path in include_path: + # Assumes that path is a full path. + # If not found, assume the included yaml + # is in the same dir as the original yaml + if not os.path.isfile(path): + path = os.path.join(yaml_dir, path) + + try: + included_yaml_config = load_yaml_config(yaml_path=path, mode=mode) + final_yaml_config.update(included_yaml_config) + except Exception as ex: + # If failed to load, ignore + raise ex + + final_yaml_config.update(yaml_config) + return final_yaml_config + return yaml_config + + +def regex_replace(string, pattern, repl, count: int = 0): + """Implements the `re.sub` function as a custom Jinja filter.""" + return re.sub(pattern, repl, string, count=count) + + +env = Environment( + loader=BaseLoader, undefined=StrictUndefined, keep_trailing_newline=True +) +env.filters["regex_replace"] = regex_replace + + +def apply_template(template: str, doc: dict) -> str: + rtemplate = env.from_string(template) + return rtemplate.render(**doc) + + +def create_iterator(raw_iterator, *, rank=0, world_size=1, limit=None): + """ + Method for creating a (potentially) sliced and limited + iterator from a raw document iterator. Used for splitting data + among ranks in multigpu setting or only pulling a sample of documents + """ + return islice(raw_iterator, rank, limit, world_size) + + +def weighted_f1_score(items): + from sklearn.metrics import f1_score + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + fscore = f1_score(golds, preds, average="weighted") + return fscore diff --git a/lm-evaluation-harness/load_dataset.py b/lm-evaluation-harness/load_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..18f617e719513cb221fddc71b55caa423cbe90d3 --- /dev/null +++ b/lm-evaluation-harness/load_dataset.py @@ -0,0 +1,19 @@ +from datasets import load_dataset + +# ๆ›ฟๆขๆˆไฝ ่ฆ็”จ็š„ไปปๅŠกๅ็งฐ +datasets_to_download = [ + "winogrande", + "hellaswag", + "arc", + "lambada_openai", + "piqa", + "trivia_qa" +] + +for name in datasets_to_download: + print(f"Downloading dataset: {name}") + try: + # ๅŠ ่ฝฝๅนถ็ผ“ๅญ˜ๆ•ฐๆฎ้›†๏ผˆ้ป˜่ฎค็ผ“ๅญ˜่ทฏๅพ„ ~/.cache/huggingface/datasets๏ผ‰ + load_dataset(name) + except Exception as e: + print(f"Failed to download {name}: {e}") diff --git a/lm-evaluation-harness/mypy.ini b/lm-evaluation-harness/mypy.ini new file mode 100644 index 0000000000000000000000000000000000000000..76a0c86452e1943edb6680b9a1fdc9627e2f7593 --- /dev/null +++ b/lm-evaluation-harness/mypy.ini @@ -0,0 +1,29 @@ +[mypy] +python_version = 3.8 +show_traceback = True +check_untyped_defs = True +no_implicit_reexport = True +warn_unreachable = True +warn_unused_configs = True +warn_unused_ignores = True +warn_redundant_casts = True + +# We ignore errors everywhere to gradually add type annotations + +[mypy-lm_eval.*] +ignore_errors = True + +[mypy-lm_eval.api.*] +ignore_errors = True + +[mypy-lm_eval.prompts.*] +ignore_errors = True + +[mypy-lm_eval.models.*] +ignore_errors = True + +[mypy-scripts.*] +ignore_errors = True + +[mypy-main] +ignore_errors = True diff --git a/lm-evaluation-harness/pile_statistics.json b/lm-evaluation-harness/pile_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..116f0eb976d735bdf92cf06341f2483e69b67e36 --- /dev/null +++ b/lm-evaluation-harness/pile_statistics.json @@ -0,0 +1,37 @@ +{ + "Data": "Pile statistics", + "Document Count": 210607728, + "Total Pile Characters": 421215456, + "File Start Offsets": [ + 0, + 7021438, + 14042822, + 21066113, + 28086515, + 35106072, + 42123306, + 49145091, + 56165817, + 63185587, + 70211208, + 77234322, + 84249267, + 91267634, + 98285983, + 105305110, + 112322489, + 119342491, + 126367373, + 133389153, + 140412039, + 147432373, + 154452516, + 161470190, + 168492733, + 175512521, + 182526939, + 189547478, + 196565318, + 203583306 + ] +} diff --git a/lm-evaluation-harness/pyproject.toml b/lm-evaluation-harness/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..45dd44189d024862d8365ad8efc0e8241ddab0f9 --- /dev/null +++ b/lm-evaluation-harness/pyproject.toml @@ -0,0 +1,141 @@ +[build-system] +requires = ["setuptools>=40.8.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "lm_eval" +version = "0.4.8" +authors = [ + {name="EleutherAI", email="contact@eleuther.ai"} +] +description = "A framework for evaluating language models" +readme = "README.md" +classifiers = [ + "Development Status :: 3 - Alpha", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +requires-python = ">=3.9" +license = { "text" = "MIT" } +dependencies = [ + "accelerate>=0.26.0", + "evaluate", + "datasets>=2.16.0", + "evaluate>=0.4.0", + "jsonlines", + "numexpr", + "peft>=0.2.0", + "pybind11>=2.6.2", + "pytablewriter", + "rouge-score>=0.0.4", + "sacrebleu>=1.5.0", + "scikit-learn>=0.24.1", + "sqlitedict", + "torch>=1.8", + "tqdm-multiprocess", + "transformers>=4.1", + "zstandard", + "dill", + "word2number", + "more_itertools", +] + +[tool.setuptools.packages.find] +include = ["lm_eval*"] + +# required to include yaml files in pip installation +[tool.setuptools.package-data] +lm_eval = ["**/*.yaml", "tasks/**/*"] + +[project.scripts] +lm-eval = "lm_eval.__main__:cli_evaluate" +lm_eval = "lm_eval.__main__:cli_evaluate" + +[project.urls] +Homepage = "https://github.com/EleutherAI/lm-evaluation-harness" +Repository = "https://github.com/EleutherAI/lm-evaluation-harness" + +[project.optional-dependencies] +acpbench = ["lark>=1.1.9", "tarski[clingo]==0.8.2", "pddl==0.4.2", "kstar-planner==1.4.2"] +api = ["requests", "aiohttp", "tenacity", "tqdm", "tiktoken"] +audiolm_qwen = ["librosa", "soundfile"] +deepsparse = ["deepsparse-nightly[llm]>=1.8.0.20240404"] +dev = ["pytest", "pytest-cov", "pytest-xdist", "pre-commit", "mypy", "unitxt==1.22.0", "requests", "aiohttp", "tenacity", "tqdm", "tiktoken", "sentencepiece"] +gptq = ["auto-gptq[triton]>=0.6.0"] +gptqmodel = ["gptqmodel>=1.0.9"] +hf_transfer = ["hf_transfer"] +ibm_watsonx_ai = ["ibm_watsonx_ai>=1.1.22", "python-dotenv"] +ifeval = ["langdetect", "immutabledict", "nltk>=3.9.1"] +ipex = ["optimum"] +japanese_leaderboard = ["emoji==2.14.0", "neologdn==0.5.3", "fugashi[unidic-lite]", "rouge_score>=0.1.2"] +longbench=["jieba", "fuzzywuzzy", "rouge"] +mamba = ["mamba_ssm", "causal-conv1d==1.0.2", "torch"] +math = ["sympy>=1.12", "antlr4-python3-runtime==4.11", "math_verify[antlr4_11_0]"] +multilingual = ["nagisa>=0.2.7", "jieba>=0.42.1", "pycountry"] +neuronx = ["optimum[neuronx]"] +optimum = ["optimum[openvino]"] +promptsource = ["promptsource>=0.2.3"] +ruler = ["nltk", "wonderwords", "scipy"] +sae_lens = ["sae_lens"] +sentencepiece = ["sentencepiece>=0.1.98"] +sparseml = ["sparseml-nightly[llm]>=1.8.0.20240404"] +sparsify = ["sparsify"] +testing = ["pytest", "pytest-cov", "pytest-xdist"] +vllm = ["vllm>=0.4.2"] +wandb = ["wandb>=0.16.3", "pandas", "numpy"] +zeno = ["pandas", "zeno-client"] +all = [ + "lm_eval[acpbench]", + "lm_eval[api]", + "lm_eval[audiolm_qwen]", + "lm_eval[deepsparse]", + "lm_eval[dev]", + "lm_eval[gptq]", + "lm_eval[gptqmodel]", + "lm_eval[hf_transfer]", + "lm_eval[ibm_watsonx_ai]", + "lm_eval[ifeval]", + "lm_eval[ipex]", + "lm_eval[japanese_leaderboard]", + "lm_eval[longbench]", + "lm_eval[mamba]", + "lm_eval[math]", + "lm_eval[multilingual]", + "lm_eval[neuronx]", + "lm_eval[optimum]", + "lm_eval[promptsource]", + "lm_eval[ruler]", + "lm_eval[sae_lens]", + "lm_eval[sentencepiece]", + "lm_eval[sparseml]", + "lm_eval[sparsify]", + "lm_eval[testing]", + "lm_eval[vllm]", + "lm_eval[wandb]", + "lm_eval[zeno]", +] + +[tool.pymarkdown] +plugins.md013.enabled = false # line-length +plugins.md024.allow_different_nesting = true # no-duplicate-headers +plugins.md025.enabled = false # single-header +plugins.md028.enabled = false # no-blanks-blockquote +plugins.md029.allow_extended_start_values = true # ol-prefix +plugins.md034.enabled = false # no-bare-urls + +[tool.ruff.lint] +extend-select = ["I"] + +[tool.ruff.lint.isort] +lines-after-imports = 2 +known-first-party = ["lm_eval"] + +[tool.ruff.lint.extend-per-file-ignores] +"__init__.py" = ["F401","F402","F403"] +"utils.py" = ["F401"] + +[dependency-groups] +dev = [ + "api","dev","sentencepiece" +] diff --git a/lm-evaluation-harness/qwen2_5_7b_gsm8k_eval.log b/lm-evaluation-harness/qwen2_5_7b_gsm8k_eval.log new file mode 100644 index 0000000000000000000000000000000000000000..69a7b411790818da39f7001b3d1724d0b8fd71d3 --- /dev/null +++ b/lm-evaluation-harness/qwen2_5_7b_gsm8k_eval.log @@ -0,0 +1,1974 @@ + +================================================== +ๅผ€ๅง‹่ฏ„ไผฐ๏ผšไปปๅŠก=gsm8k | ๅฐ‘ๆ ทๆœฌๆ•ฐ=4 | ๆจกๅž‹=Qwen2.5-7B +่พ“ๅ‡บ่ทฏๅพ„๏ผšresults2/Qwen2.5-7B/base_gsm8k.json +================================================== +2025-12-01:20:07:52 INFO [__main__:440] Selected Tasks: ['gsm8k'] +2025-12-01:20:07:52 INFO [evaluator:189] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234 +2025-12-01:20:07:52 INFO [evaluator:227] Initializing hf model, with arguments: {'pretrained': '/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen/Qwen2.5-7B'} +2025-12-01:20:07:52 WARNING [accelerate.utils.other:513] Detected kernel version 5.4.143, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. +2025-12-01:20:07:52 INFO [models.huggingface:137] Using device 'cuda:3' +2025-12-01:20:07:52 INFO [models.huggingface:382] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:3'} +`torch_dtype` is deprecated! Use `dtype` instead! + +Loading checkpoint shards: 0%| | 0/4 [00:00', '<|im_end|>'], 'do_sample': False, 'temperature': 0.0} +2025-12-01:20:08:17 WARNING [evaluator:309] Overwriting default num_fewshot of gsm8k from 5 to 4 +2025-12-01:20:08:17 INFO [api.task:434] Building contexts for gsm8k on rank 0... + + 0%| | 0/1319 [00:00