Add files using upload-large-folder tool

lm-evaluation-harness/lm_eval/models/hf_steered.py

@@ -0,0 +1,243 @@
+from contextlib import contextmanager
+from functools import partial
+from pathlib import Path
+from typing import Any, Callable, Generator, Optional, Union
+
+import torch
+from peft.peft_model import PeftModel
+from torch import Tensor, nn
+from transformers import PreTrainedModel
+
+from lm_eval.api.registry import register_model
+from lm_eval.models.huggingface import HFLM
+
+
+@contextmanager
+def steer(
+    model: Union[PreTrainedModel, PeftModel], hook_to_steer: dict[str, Callable]
+) -> Generator[None, Any, None]:
+    """
+    Context manager that temporarily hooks models and steers them.
+
+    Args:
+        model: The transformer model to hook
+        hook_to_steer: Dictionary mapping hookpoints to steering functions
+
+    Yields:
+        None
+    """
+
+    def create_hook(hookpoint: str):
+        def hook_fn(module: nn.Module, input: Any, output: Tensor):
+            # If output is a tuple (like in some transformer layers), take first element
+            if isinstance(output, tuple):
+                output = (hook_to_steer[hookpoint](output[0]), *output[1:])  # type: ignore
+            else:
+                output = hook_to_steer[hookpoint](output)
+
+            return output
+
+        return hook_fn
+
+    handles = []
+    hookpoints = list(hook_to_steer.keys())
+
+    for name, module in model.base_model.named_modules():
+        if name in hookpoints:
+            handle = module.register_forward_hook(create_hook(name))
+            handles.append(handle)
+
+    if len(handles) != len(hookpoints):
+        raise ValueError(f"Not all hookpoints could be resolved: {hookpoints}")
+
+    try:
+        yield None
+    finally:
+        for handle in handles:
+            handle.remove()
+
+
+@register_model("steered")
+class SteeredModel(HFLM):
+    hook_to_steer: dict[str, Callable]
+
+    def __init__(
+        self,
+        pretrained: str,
+        steer_path: str,
+        device: Optional[str] = None,
+        **kwargs,
+    ):
+        """
+        HFLM with a steered forward pass.
+
+        To derive steering vectors from a sparse model loadable with sparsify or sae_lens,
+        provide the path to a CSV file with the following columns (example rows are provided below):
+
+        loader,action,sparse_model,hookpoint,feature_index,steering_coefficient,sae_id,description,
+        sparsify,add,EleutherAI/sae-pythia-70m-32k,layers.3,30,10.0,,,
+        sae_lens,add,gemma-scope-2b-pt-res-canonical,layers.20,12082,240.0,layer_20/width_16k/canonical,increase dogs,
+
+        To load steering vectors directly, provide the path to a pytorch (.pt) file with content in the following format:
+
+        {
+            hookpoint: {
+                "steering_vector": <torch.Tensor>,
+                "steering_coefficient": <float>,
+                "action": <Literal["add", "clamp"]>,
+                "bias": <torch.Tensor | None>,
+            },
+            ...
+        }
+        """
+        super().__init__(pretrained=pretrained, device=device, **kwargs)
+
+        if steer_path.endswith(".pt") or steer_path.endswith(".pth"):
+            with open(steer_path, "rb") as f:
+                steer_config: dict[str, dict[str, Any]] = torch.load(
+                    f, weights_only=True
+                )
+        elif steer_path.endswith(".csv"):
+            steer_config = self.derive_steer_config(steer_path)
+        else:
+            raise ValueError(f"Unknown steer file type: {steer_path}")
+
+        hook_to_steer = {}
+        for hookpoint, steer_info in steer_config.items():
+            action = steer_info["action"]
+            steering_coefficient = steer_info["steering_coefficient"]
+            steering_vector = (
+                steer_info["steering_vector"].to(self.device).to(self.model.dtype)
+            )
+            bias = (
+                steer_info["bias"].to(self.device).to(self.model.dtype)
+                if steer_info["bias"] is not None
+                else None
+            )
+
+            if action == "add":
+                # Steers the model by adding some multiple of a steering vector to all sequence positions.
+                hook_to_steer[hookpoint] = (
+                    lambda acts: acts + steering_coefficient * steering_vector
+                )
+            elif action == "clamp":
+                hook_to_steer[hookpoint] = partial(
+                    self.clamp,
+                    steering_vector=steering_vector,
+                    value=steering_coefficient,
+                    bias=bias,
+                )
+            else:
+                raise ValueError(f"Unknown hook type: {action}")
+
+        self.hook_to_steer = hook_to_steer
+
+    @classmethod
+    def derive_steer_config(cls, steer_path: str):
+        """Derive a dictionary of steering vectors from sparse model(/s) specified in a CSV file."""
+        import pandas as pd
+
+        df = pd.read_csv(steer_path)
+        steer_data: dict[str, dict[str, Any]] = {}
+
+        if any(df["loader"] == "sparsify"):
+            from sparsify import SparseCoder
+        if any(df["loader"] == "sae_lens"):
+            from sae_lens import SAE
+
+            sae_cache = {}
+
+            def load_from_sae_lens(sae_release: str, sae_id: str):
+                cache_key = (sae_release, sae_id)
+                if cache_key not in sae_cache:
+                    sae_cache[cache_key] = SAE.from_pretrained(sae_release, sae_id)[0]
+
+                return sae_cache[cache_key]
+
+        for _, row in df.iterrows():
+            action = row.get("action", "add")
+            sparse_name = row["sparse_model"]
+            hookpoint = row["hookpoint"]
+            feature_index = int(row["feature_index"])
+            steering_coefficient = float(row["steering_coefficient"])
+            loader = row.get("loader", "sparsify")
+
+            if loader == "sparsify":
+                name_path = Path(sparse_name)
+
+                sparse_coder = (
+                    SparseCoder.load_from_disk(name_path / hookpoint)
+                    if name_path.exists()
+                    else SparseCoder.load_from_hub(sparse_name, hookpoint)
+                )
+                assert sparse_coder.W_dec is not None
+
+                steering_vector = sparse_coder.W_dec[feature_index]
+                bias = sparse_coder.b_dec
+
+            elif loader == "sae_lens":
+                sparse_coder = load_from_sae_lens(
+                    sae_release=sparse_name, sae_id=row["sae_id"]
+                )
+                steering_vector = sparse_coder.W_dec[feature_index]
+                bias = sparse_coder.b_dec
+                if hookpoint == "" or pd.isna(hookpoint):
+                    hookpoint = sparse_coder.cfg.hook_name
+            else:
+                raise ValueError(f"Unknown loader: {loader}")
+
+            steer_data[hookpoint] = {
+                "action": action,
+                "steering_coefficient": steering_coefficient,
+                "steering_vector": steering_vector,
+                "bias": bias,
+            }
+
+        return steer_data
+
+    @classmethod
+    def clamp(
+        cls,
+        acts: Tensor,
+        steering_vector: Tensor,
+        value: float,
+        bias: Optional[Tensor] = None,
+    ):
+        """Clamps a direction of the activations to be the steering vector * the value.
+
+        Args:
+            acts (Tensor): The activations tensor to edit of shape [batch, pos, features]
+            steering_vector (Tensor): A direction to clamp of shape [features]
+            value (float): Value to clamp the direction to
+            bias (Tensor | None): Optional bias to add to the activations
+
+        Returns:
+            Tensor: The modified activations with the specified direction clamped
+        """
+
+        if bias is not None:
+            acts = acts - bias
+
+        direction = steering_vector / torch.norm(steering_vector)
+        proj_magnitude = torch.sum(acts * direction, dim=-1, keepdim=True)
+        orthogonal_component = acts - proj_magnitude * direction
+
+        clamped = orthogonal_component + direction * value
+
+        if bias is not None:
+            return clamped + bias
+
+        return clamped
+
+    def forward(self, *args, **kwargs):
+        with torch.no_grad():
+            with steer(self.model, self.hook_to_steer):
+                return self.model.forward(*args, **kwargs)
+
+    def _model_call(self, *args, **kwargs):
+        with steer(self.model, self.hook_to_steer):
+            return super()._model_call(*args, **kwargs)
+
+    def _model_generate(self, *args, **kwargs):
+        with steer(self.model, self.hook_to_steer):
+            return super()._model_generate(*args, **kwargs)

lm-evaluation-harness/lm_eval/models/openai_completions.py

@@ -0,0 +1,296 @@
+import logging
+import os
+from functools import cached_property
+from operator import itemgetter
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from lm_eval.api.registry import register_model
+from lm_eval.models.api_models import TemplateAPI
+from lm_eval.models.utils import handle_stop_sequences
+
+
+eval_logger = logging.getLogger(__name__)
+
+
+@register_model("local-completions")
+class LocalCompletionsAPI(TemplateAPI):
+    def __init__(
+        self,
+        base_url=None,
+        tokenizer_backend="huggingface",
+        **kwargs,
+    ):
+        super().__init__(
+            base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs
+        )
+
+    def _create_payload(
+        self,
+        messages: Union[List[List[int]], List[dict], List[str], str],
+        generate=False,
+        gen_kwargs: Optional[dict] = None,
+        seed: int = 1234,
+        eos=None,
+        **kwargs,
+    ) -> dict:
+        if generate:
+            gen_kwargs.pop("do_sample", False)
+            if "max_tokens" in gen_kwargs:
+                max_tokens = gen_kwargs.pop("max_tokens")
+            else:
+                max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks)
+            temperature = gen_kwargs.pop("temperature", 0)
+            stop = handle_stop_sequences(gen_kwargs.pop("until", None), eos)
+            return {
+                "prompt": messages,
+                "model": self.model,
+                "max_tokens": max_tokens,
+                "temperature": temperature,
+                "stop": stop,
+                "seed": seed,
+                **gen_kwargs,
+            }
+        else:
+            return {
+                "model": self.model,
+                "prompt": messages,
+                "temperature": 0,
+                "max_tokens": 1,
+                "logprobs": 1,
+                "seed": seed,
+                "echo": True,
+            }
+
+    @staticmethod
+    def parse_logprobs(
+        outputs: Union[Dict, List[Dict]],
+        tokens: List[List[int]] = None,
+        ctxlens: List[int] = None,
+        **kwargs,
+    ) -> List[Tuple[float, bool]]:
+        res = []
+        if not isinstance(outputs, list):
+            outputs = [outputs]
+        for out in outputs:
+            for choice, ctxlen in zip(
+                sorted(out["choices"], key=itemgetter("index")), ctxlens
+            ):
+                assert ctxlen > 0, "Context length must be greater than 0"
+                logprobs = sum(choice["logprobs"]["token_logprobs"][ctxlen:-1])
+                tokens_logprobs = choice["logprobs"]["token_logprobs"][ctxlen:-1]
+                top_logprobs = choice["logprobs"]["top_logprobs"][ctxlen:-1]
+                is_greedy = True
+                for tok, top in zip(tokens_logprobs, top_logprobs):
+                    if tok != max(top.values()):
+                        is_greedy = False
+                        break
+                res.append((logprobs, is_greedy))
+        return res
+
+    @staticmethod
+    def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]:
+        res = []
+        if not isinstance(outputs, list):
+            outputs = [outputs]
+        for out in outputs:
+            tmp = [None] * len(out["choices"])
+            for choices in out["choices"]:
+                tmp[choices["index"]] = choices["text"]
+            res = res + tmp
+        return res
+
+    @property
+    def api_key(self):
+        return os.environ.get("OPENAI_API_KEY", "")
+
+
+@register_model("local-chat-completions")
+class LocalChatCompletion(LocalCompletionsAPI):
+    def __init__(
+        self,
+        base_url=None,
+        tokenizer_backend=None,
+        tokenized_requests=False,
+        **kwargs,
+    ):
+        eval_logger.warning(
+            "chat-completions endpoint requires the `--apply_chat_template` flag."
+        )
+        super().__init__(
+            base_url=base_url,
+            tokenizer_backend=tokenizer_backend,
+            tokenized_requests=tokenized_requests,
+            **kwargs,
+        )
+        if self._batch_size > 1:
+            eval_logger.warning(
+                "Chat completions does not support batching. Defaulting to batch size 1."
+            )
+            self._batch_size = 1
+
+    def _create_payload(
+        self,
+        messages: List[Dict],
+        generate=False,
+        gen_kwargs: dict = None,
+        seed=1234,
+        eos=None,
+        **kwargs,
+    ) -> dict:
+        assert type(messages) is not str, (
+            "chat-completions require the --apply_chat_template flag."
+        )
+        gen_kwargs.pop("do_sample", False)
+        if "max_tokens" in gen_kwargs:
+            max_tokens = gen_kwargs.pop("max_tokens")
+        else:
+            max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks)
+        temperature = gen_kwargs.pop("temperature", 0)
+        stop = handle_stop_sequences(gen_kwargs.pop("until", None), eos)
+        if not isinstance(stop, (list, tuple)):
+            stop = [stop]
+        return {
+            "messages": messages,
+            "model": self.model,
+            "max_tokens": max_tokens,
+            "temperature": temperature,
+            "stop": stop[:4],
+            "seed": seed,
+            **gen_kwargs,
+        }
+
+    @staticmethod
+    def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]:
+        res = []
+        if not isinstance(outputs, list):
+            outputs = [outputs]
+        for out in outputs:
+            tmp = [None] * len(out["choices"])
+            for choices in out["choices"]:
+                tmp[choices["index"]] = choices["message"]["content"]
+            res = res + tmp
+        return res
+
+    def tok_encode(
+        self,
+        string: Union[str, Any],
+        left_truncate_len=None,
+        add_special_tokens=None,
+        **kwargs,
+    ) -> Union[List[str], List[int], Any]:
+        return string
+
+    def loglikelihood(self, requests, **kwargs):
+        raise NotImplementedError(
+            "Loglikelihood is not supported for chat completions. Consider using the completions API instead."
+        )
+
+
+@register_model(
+    "openai-completions",
+)
+class OpenAICompletionsAPI(LocalCompletionsAPI):
+    def __init__(
+        self,
+        base_url="https://api.openai.com/v1/completions",
+        tokenizer_backend="tiktoken",
+        **kwargs,
+    ):
+        super().__init__(
+            base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs
+        )
+
+    @cached_property
+    def api_key(self):
+        """Override this property to return the API key for the API request."""
+        key = os.environ.get("OPENAI_API_KEY", None)
+        if key is None:
+            raise ValueError(
+                "API key not found. Please set the `OPENAI_API_KEY` environment variable."
+            )
+        return key
+
+    def loglikelihood(self, requests, **kwargs):
+        assert self.model in [
+            "babbage-002",
+            "davinci-002",
+        ], (
+            f"Prompt loglikelihoods are only supported by OpenAI's API for {['babbage-002', 'davinci-002']}."
+        )
+        return super().loglikelihood(requests, **kwargs)
+
+    def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]:
+        return ""
+
+
+@register_model("openai-chat-completions")
+class OpenAIChatCompletion(LocalChatCompletion):
+    def __init__(
+        self,
+        base_url="https://api.openai.com/v1/chat/completions",
+        tokenizer_backend=None,
+        tokenized_requests=False,
+        **kwargs,
+    ):
+        if "o1" in kwargs.get("model", ""):
+            eval_logger.warning(
+                "o1 models do not support `stop` and only support temperature=1"
+            )
+        super().__init__(
+            base_url=base_url,
+            tokenizer_backend=tokenizer_backend,
+            tokenized_requests=tokenized_requests,
+            **kwargs,
+        )
+
+    @cached_property
+    def api_key(self):
+        """Override this property to return the API key for the API request."""
+        key = os.environ.get("OPENAI_API_KEY", None)
+        if key is None:
+            raise ValueError(
+                "API key not found. Please set the `OPENAI_API_KEY` environment variable."
+            )
+        return key
+
+    def loglikelihood(self, requests, **kwargs):
+        raise NotImplementedError(
+            "Loglikelihood (and therefore `multiple_choice`-type tasks) is not supported for chat completions as OpenAI does not provide prompt logprobs. See https://github.com/EleutherAI/lm-evaluation-harness/issues/942#issuecomment-1777836312 or https://github.com/EleutherAI/lm-evaluation-harness/issues/1196 for more background on this limitation."
+        )
+
+    def _create_payload(
+        self,
+        messages: List[Dict],
+        generate=False,
+        gen_kwargs: dict = None,
+        seed=1234,
+        eos="<|endoftext|>",
+        **kwargs,
+    ) -> dict:
+        assert type(messages) is not str, (
+            "chat-completions require the --apply_chat_template flag."
+        )
+        gen_kwargs.pop("do_sample", False)
+        if "max_tokens" in gen_kwargs:
+            max_tokens = gen_kwargs.pop("max_tokens")
+        else:
+            max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks)
+        temperature = gen_kwargs.pop("temperature", 0)
+        stop = handle_stop_sequences(gen_kwargs.pop("until", ["<|endoftext|>"]), eos)
+        if not isinstance(stop, (list, tuple)):
+            stop = [stop]
+        output = {
+            "messages": messages,
+            "model": self.model,
+            "max_completion_tokens": max_tokens,
+            "temperature": temperature,
+            "stop": stop[:4],
+            "seed": seed,
+            **gen_kwargs,
+        }
+        if "o1" in self.model:
+            output.pop("stop")
+            output["temperature"] = 1
+        elif "o3" in self.model:
+            output.pop("temperature")
+        return output

lm-evaluation-harness/lm_eval/models/utils.py

@@ -0,0 +1,854 @@
+import collections
+import fnmatch
+import gc
+import itertools
+import logging
+import time
+from functools import wraps
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+)
+
+import torch
+import transformers
+
+
+eval_logger = logging.getLogger(__name__)
+
+
+if TYPE_CHECKING:
+    from PIL import Image
+    from transformers import PreTrainedTokenizerBase
+    from transformers.configuration_utils import PretrainedConfig
+
+
+def chunks(iter, n: int = 0, fn=None):
+    """
+    Divides an iterable into chunks of specified size or based on a given function.
+    Useful for batching
+
+    Parameters:
+    - iter: The input iterable to be divided into chunks.
+    - n: An integer representing the size of each chunk. Default is 0.
+    - fn: A function that takes the current index and the iterable as arguments and returns the size of the chunk. Default is None.
+
+    Returns:
+    An iterator that yields chunks of the input iterable.
+
+    Example usage:
+    ```
+    data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+    for chunk in chunks(data, 3):
+        print(chunk)
+    ```
+    Output:
+    ```
+    [1, 2, 3]
+    [4, 5, 6]
+    [7, 8, 9]
+    [10]
+    ```
+    """
+    arr = []
+    for i, x in enumerate(iter):
+        arr.append(x)
+        if len(arr) == (fn(i, iter) if fn else n):
+            yield arr
+            arr = []
+
+    if arr:
+        yield arr
+
+
+class MultiChoice:
+    def __init__(self, choices) -> None:
+        self.choices = choices
+
+    # Simple wildcard support (linux filename patterns)
+    def __contains__(self, values) -> bool:
+        for value in values.split(","):
+            if len(fnmatch.filter(self.choices, value)) == 0:
+                eval_logger.info("Available tasks to choose:")
+                for choice in self.choices:
+                    eval_logger.info(f"  - {choice}")
+                raise ValueError("'{}' is not in task list".format(value))
+        return True
+
+    def __iter__(self) -> Iterator:
+        for choice in self.choices:
+            yield choice
+
+
+class Grouper:
+    """
+    takes an array `arr` and function `fn` and returns a dictionary
+    with keys fn(ob) for each ob in `arr` and with values `self.arr[key]` a list of all
+    objects in `arr` satisfying `key == fn(ob)`.
+    """
+
+    def __init__(self, arr, fn) -> None:
+        # self.orig_arr = arr
+        self.size = len(arr)
+        arr = list(enumerate(arr))
+
+        def group_return_dict(arr, fn):
+            res = collections.defaultdict(list)
+
+            for ob in arr:
+                res[fn(ob)].append(ob)
+            return res
+
+        arr = group_return_dict(arr, lambda x: fn(x[1]))
+
+        # self.arr has format Dict[Tuple[int, <entry from orig. arr>]]
+        self.arr = arr
+        self._grouped = None
+
+    def get_grouped(self):
+        # return the contents but not indices for our grouped dict.
+        if self._grouped:
+            return self._grouped
+        grouped = {}
+        for key in self.arr.keys():
+            # drop the index from each element of self.arr
+            grouped[key] = [y[1] for y in self.arr[key]]
+        self._grouped = grouped
+        return grouped
+
+    def get_original(self, grouped_dict):
+        # take in a grouped dictionary with e.g. results for each key listed
+        # in the same order as the instances in `self.arr`, and
+        # return the results in the same (single list) order as `self.orig_arr`.
+        res = [None] * self.size
+        cov = [False] * self.size
+        # orig = [None] * self.size
+
+        assert grouped_dict.keys() == self.arr.keys()
+
+        for key in grouped_dict.keys():
+            for (ind, _), v in zip(self.arr[key], grouped_dict[key]):
+                res[ind] = v
+                cov[ind] = True
+                # orig[ind] = _
+
+        assert all(cov)
+        # assert orig == self.orig_arr
+
+        return res
+
+
+def pad_and_concat(
+    max_length: int,
+    tensors: List[torch.Tensor],
+    padding_side: Literal["right", "left"] = "right",
+):
+    """
+    Method for padding a list of tensors given the maximum tensor
+    length in the batch. Used for batching inputs and continuations in
+    seq2seq models.
+    """
+    assert padding_side == "left" or padding_side == "right", (
+        f"Unrecognized padding type: '{padding_side}' not 'left' or 'right'"
+    )
+
+    for i, tensor in enumerate(tensors):
+        if len(tensor.shape) == 2:
+            tensor = tensor.squeeze(0)  # squeeze, in case passed [1, seq] size
+        tensor_len = tensor.shape[0]
+        if tensor_len < max_length:
+            if padding_side == "right":
+                # right-pad
+                tensors[i] = torch.cat(
+                    [
+                        tensor,  # [seq]
+                        torch.zeros(
+                            max_length - tensor_len,
+                            dtype=torch.long,
+                            device=tensor.device,
+                        ),  # [padding_length - seq]
+                    ],
+                    dim=0,
+                ).unsqueeze(0)
+            else:
+                # left-pad
+                tensors[i] = torch.cat(
+                    [
+                        torch.zeros(
+                            max_length - tensor_len,
+                            dtype=torch.long,
+                            device=tensor.device,
+                        ),  # [padding_length - seq]
+                        tensor,  # [seq]
+                    ],
+                    dim=0,
+                ).unsqueeze(0)
+        else:
+            tensors[i] = tensor.unsqueeze(0)
+
+    return torch.cat(tensors, dim=0)
+
+
+def clear_torch_cache() -> None:
+    gc.collect()
+    torch.cuda.empty_cache()
+
+
+def get_dtype(dtype: Union[str, torch.dtype]) -> torch.dtype:
+    """Converts `dtype` from `str` to torch.dtype when possible. Does not use an instantiated HF AutoConfig"""
+    if isinstance(dtype, str) and dtype != "auto":
+        # Convert `str` args torch dtype: `float16` -> `torch.float16`
+        _torch_dtype = getattr(torch, dtype)
+    else:
+        _torch_dtype = dtype
+    return _torch_dtype
+
+
+class MultiTokenEOSCriteria(transformers.StoppingCriteria):
+    """Criteria to stop on the specified multi-token sequence."""
+
+    def __init__(
+        self,
+        sequence: str,
+        tokenizer: transformers.PreTrainedTokenizer,
+        initial_decoder_input_length: int,
+        batch_size: int,
+    ) -> None:
+        self.initial_decoder_input_length = initial_decoder_input_length
+        self.done_tracker = [False] * batch_size
+        self.sequence = sequence
+        self.sequence_ids = tokenizer.encode(sequence, add_special_tokens=False)
+        # print(sequence, self.sequence_ids)
+        # we look back for 2 more tokens than it takes to encode our stop sequence
+        # because tokenizers suck, and a model might generate `['\n', '\n']` but our `sequence` is `['\n\n']`
+        # and we don't want to mistakenly not stop a generation because our
+        # (string) stop sequence was output in a different tokenization
+
+        # NOTE: there is a minor danger that this will end up looking back 2 tokens into the past, into the inputs to the model,
+        # and stopping generation immediately as a result. With only 2 extra tokens of lookback, this risk is minimized
+        # Additionally, in lookback_ids_batch we should prevent ever looking back into the inputs as described.
+        self.sequence_id_len = len(self.sequence_ids) + 2
+        self.tokenizer = tokenizer
+
+    def __call__(self, input_ids, scores, **kwargs) -> bool:
+        # For efficiency, we compare the last n tokens where n is the number of tokens in the stop_sequence
+        lookback_ids_batch = input_ids[:, self.initial_decoder_input_length :]
+
+        lookback_ids_batch = lookback_ids_batch[:, -self.sequence_id_len :]
+
+        lookback_tokens_batch = self.tokenizer.batch_decode(lookback_ids_batch)
+
+        for i, done in enumerate(self.done_tracker):
+            if not done:
+                self.done_tracker[i] = self.sequence in lookback_tokens_batch[i]
+        return False not in self.done_tracker
+
+
+def stop_sequences_criteria(
+    tokenizer: transformers.PreTrainedTokenizer,
+    stop_sequences: List[str],
+    initial_decoder_input_length: int,
+    batch_size: int,
+) -> transformers.StoppingCriteriaList:
+    return transformers.StoppingCriteriaList(
+        [
+            *[
+                MultiTokenEOSCriteria(
+                    sequence, tokenizer, initial_decoder_input_length, batch_size
+                )
+                for sequence in stop_sequences
+            ],
+        ]
+    )
+
+
+def undistribute(iterable):
+    """
+    Undoes https://more-itertools.readthedocs.io/en/stable/api.html#more_itertools.distribute .
+
+    Re-interleaves results that have been split using more_itertools.distribute:
+        >>> group_1, group_2 = distribute(2, [1, 2, 3, 4, 5, 6])
+        >>> list(group_1)
+        [1, 3, 5]
+        >>> list(group_2)
+        [2, 4, 6]
+        >>> undistribute([group_1, group_2])
+        [1, 2, 3, 4, 5, 6]
+
+    Handles non-uniform component lengths:
+
+        >>> children = distribute(3, [1, 2, 3, 4, 5, 6, 7])
+        >>> [list(c) for c in children]
+        [[1, 4, 7], [2, 5], [3, 6]]
+        >>> undistribute(children)
+        [1, 2, 3, 4, 5, 6, 7]
+
+    Also handles when some iterables are empty:
+
+        >>> children = distribute(5, [1, 2, 3])
+        >>> [list(c) for c in children]
+        [[1], [2], [3], [], []]
+        >>> undistribute(children)
+        [1, 2, 3]
+
+    """
+
+    return [
+        x
+        for x in itertools.chain.from_iterable(
+            itertools.zip_longest(*[list(x) for x in iterable])
+        )
+        if x is not None
+    ]
+
+
+def retry_on_specific_exceptions(
+    on_exceptions: List[Type[Exception]],
+    max_retries: Optional[int] = None,
+    backoff_time: float = 3.0,
+    backoff_multiplier: float = 1.5,
+    on_exception_callback: Optional[Callable[[Exception, float], Any]] = None,
+):
+    """Retry on an LLM Provider's rate limit error with exponential backoff
+    For example, to use for OpenAI, do the following:
+    ```
+    from openai import RateLimitError
+
+    # Recommend specifying max_retries to avoid infinite loops!
+    @retry_on_specific_exceptions([RateLimitError], max_retries=3)
+    def completion(...):
+        # Wrap OpenAI completion function here
+        ...
+    ```
+    """
+
+    def decorator(func: Callable):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            sleep_time = backoff_time
+            attempt = 0
+            while max_retries is None or attempt < max_retries:
+                try:
+                    return func(*args, **kwargs)
+                except tuple(on_exceptions) as e:
+                    if on_exception_callback is not None:
+                        on_exception_callback(e, sleep_time)
+                    time.sleep(sleep_time)
+                    sleep_time *= backoff_multiplier
+                    attempt += 1
+
+        return wrapper
+
+    return decorator
+
+
+class Collator:
+    """
+    A class for reordering and batching elements of an array.
+
+    This class allows for sorting an array based on a provided sorting function, grouping elements based on a grouping function, and generating batches from the sorted and grouped data.
+
+    Objects of this class have the group_by attribute which determines the method for grouping
+    the data while batching it. Three options include "gen_kwargs", "contexts", or None:
+        If group_by == "gen_kwargs" then requests will be grouped by gen_kwargs
+        If group_by == "contexts" then requests will be grouped by context + cont[:-1]
+        If None then requests will just be reordered by length descending.
+    """
+
+    def __init__(
+        self,
+        arr: List,
+        sort_fn: Callable = lambda x: x,
+        group_fn: Callable = lambda x: x[1],
+        group_by: Union[Literal["gen_kwargs", "contexts"], None] = None,
+    ) -> None:
+        self._group_by = group_by
+        # 0 indices are enumerated indices. Apply functions to original arr.
+        self._sort_fn = lambda x: sort_fn(x[1])
+        self._group_fn = lambda x: group_fn(x[1])
+        self._reorder_indices: List = []
+        self._size = len(arr)
+        self._arr_with_indices: Union[Dict, Tuple[Tuple[int, Any], ...]] = tuple(
+            enumerate(arr)
+        )  # [indices, (arr)]
+        if self._group_by == "contexts":
+            self._group_by_context()
+        elif self._group_by == "gen_kwargs":
+            self._group_by_index()
+
+    def _group_by_index(self) -> None:
+        """Group the elements of a list based on their indices."""
+        self._arr_with_indices = self.group(
+            self._arr_with_indices, fn=self._group_fn, group_by="gen_kwargs"
+        )
+
+    def _group_by_context(self) -> None:
+        """Group the array with indices by context."""
+        self._arr_with_indices = self.group(
+            self._arr_with_indices, fn=self._group_fn, group_by="contexts"
+        )
+
+    def get_batched(self, n: int = 1, batch_fn: Optional[Callable] = None) -> Iterator:
+        """
+        Generates and yields batches from the reordered array. The method of grouping and batching
+        depends on the parameter `group_by`.
+        If `group_by` is set to "gen_kwargs", it will batch the
+        re-ordered values with same gen_kwargs for each batch.
+        If `group_by` is "contexts", it caches the requests by context before batching.
+        If `group_by` is neither "gen_kwargs" nor "contexts", it yields the reordered array
+
+        Parameters:
+        - n (int): The size of each batch. Defaults to 1.
+        - batch_fn ([Callable[[int, Iterable], int]] | None): A function to determine the size of
+          each batch. Optional, defaults to None.
+
+        Returns:
+        Iterator: An iterator over batches of reordered elements grouped as per the `group_by`
+                  attribute.
+
+        Yields:
+        List of batched elements according to the `group_by` attribute.
+        """
+        if self._group_by == "gen_kwargs":
+            for (
+                key,
+                values,
+            ) in self._arr_with_indices.items():  # type: ignore
+                values = self._reorder(values)
+                batch = self.get_chunks(values, n=n, fn=batch_fn)
+                yield from batch
+        elif self._group_by == "contexts":
+            # Get one sample from each key.
+            # Select longest continuation per group to ensure sufficient context logits
+            values = self._reorder(
+                [
+                    max(value, key=lambda x: len(x[1][-1]))
+                    for value in self._arr_with_indices.values()
+                ]
+            )
+            batch = self.get_chunks(values, n=n, fn=batch_fn)
+            yield from batch
+        else:
+            values = self._reorder(self._arr_with_indices)  # type: ignore
+            batch = self.get_chunks(values, n=n, fn=batch_fn)
+            yield from batch
+
+    def get_cache(
+        self,
+        req_str: Tuple[str, str] = None,
+        cxt_toks: List[int] = None,
+        cont_toks: List[int] = None,
+        logits: torch.Tensor = None,
+    ) -> Iterator[Tuple[Tuple[str, str], List[int], torch.Tensor]]:
+        """
+        Retrieves cached single-token continuations and their associated arguments, updating indices as necessary.
+
+        The behavior of this function varies depending on how the `group_by` attribute is set:
+
+        - When `group_by` is "contexts":
+            The function identifies single-token continuations by checking for keys that equate to
+            [context+continuation][-1] and logs the indices for re-ordering.
+            In this mode, this function can work in two scenarios:
+
+            1. Cache Hit - Single Match:
+                If a single matching context-continuation pair is found in the cache,
+                the function yields the original arguments.
+
+            2. Cache Hit - Multiple Matches:
+                If multiple matching context-continuation pairs are found in the cache,
+                the function expands the logits batch dimension to match the number of cache hits.
+                It updates the original requests and continuation tokens.
+
+        - When `group_by` is not set to "contexts":
+            This method yields the original arguments, logits and continuation tokens,
+            without checking for one-token continuations.
+
+        Parameters:
+        - req_str (tuple[str, str]): Original strings used for CachingLM.
+        - cxt_toks (list[int]): Full context tokens used for lookup.
+        - cont_toks (list[int]): Continuation tokens for which logits were generated.
+        - logits (torch.Tensor [1, seq_length, vocab_size]): Logits generated by the model given context and continuation keys.
+
+        Yields:
+        - Iterator:
+            - req_str (tuple[str, str]): strings used for CachingLM.
+            - cont_toks (list[int]) : continuation tokens.
+            - logits (torch.Tensor [1, seq_length, vocab_size]): The original logits (repeated cache hit times)
+        """
+        if self._group_by == "contexts":
+            cache_hit: List[
+                Tuple[int, Tuple[Tuple[str, str], List[int], List[int]]]
+            ] = self._arr_with_indices.pop(tuple(cxt_toks + cont_toks[:-1]))
+            if (cache_size := len(cache_hit)) == 1:
+                self._reorder_indices.extend(x[0] for x in cache_hit)
+                yield req_str, cont_toks, logits
+            else:
+                # If we have matching requests then expand the batch dimension (no-op) and
+                # yield each along with its corresponding args.
+                multilogits = logits.expand(cache_size, -1, -1).chunk(cache_size)
+                indices, req_str, cont_toks = zip(
+                    *[(x[0], x[1][0], x[-1][-1]) for x in cache_hit]
+                )
+                self._reorder_indices.extend(indices)
+                for c_key, cont_tok, logit in zip(req_str, cont_toks, multilogits):
+                    yield c_key, cont_tok, logit
+        else:
+            yield req_str, cont_toks, logits
+
+    def _reorder(self, arr: Union[List, Tuple[Tuple[int, Any], ...]]) -> Iterator:
+        """
+        Reorders the elements in the array based on the sorting function.
+
+        Parameters:
+        - arr (list | tuple[tuple[int, Any], ...]]): The array or iterable to be reordered.
+
+        Yields:
+            Iterator
+        """
+        arr = sorted(arr, key=self._sort_fn)
+        if not self._group_by == "contexts":
+            # If grouped by contexts then indices will be set in get_cache()
+            self._reorder_indices.extend([x[0] for x in arr])
+        yield from [x[1] for x in arr]
+
+    def get_original(self, newarr: List) -> List:
+        """
+        Restores the original order of elements from the reordered list.
+
+        Parameters:
+        - newarr (list): The reordered array.
+
+        Returns:
+        list: The array with elements restored to their original order.
+        """
+        res = [None] * self._size
+        cov = [False] * self._size
+
+        for ind, v in zip(self._reorder_indices, newarr):
+            res[ind] = v
+            cov[ind] = True
+
+        assert all(cov)
+
+        return res
+
+    def __len__(self):
+        return self._size
+
+    @staticmethod
+    def group(
+        arr: Iterable,
+        fn: Callable,
+        group_by: Literal["gen_kwargs", "contexts"] = "gen_kwargs",
+    ) -> dict:
+        """
+        Groups elements of an iterable based on a provided function.
+
+
+        The `group_by` parameter determines the method of grouping.
+        If `group_by` is "contexts", the elements are grouped by [context + cont][:-1].
+        If `group_by` is "gen_kwargs", the elements are grouped based on the gen_kwargs dict.
+
+        Parameters:
+        - arr (Iterable): The iterable to be grouped.
+        - fn (Callable): The function to determine the grouping.
+        - values (bool): If True, returns the values of the group. Defaults to False.
+
+        Returns:
+        Iterator: An iterable of grouped elements.
+        """
+        res = collections.defaultdict(list)
+        for ob in arr:
+            # where ob == [context + cont]
+            if group_by == "contexts":
+                res[tuple(fn(ob))].append(ob)
+            else:
+                try:
+                    hashable_dict = tuple(
+                        (
+                            key,
+                            tuple(value)
+                            if isinstance(value, collections.abc.Iterable)
+                            else value,
+                        )
+                        for key, value in sorted(fn(ob).items())
+                    )
+                    res[hashable_dict].append(ob)
+                except (TypeError, AttributeError):
+                    res[tuple(fn(ob))].append(ob)
+        return res
+
+    @staticmethod
+    def get_chunks(_iter, n: int = 0, fn=None):
+        """
+        Divides an iterable into chunks of specified size or based on a given function.
+        Useful for batching
+
+        Parameters:
+        - iter: The input iterable to be divided into chunks.
+        - n: An integer representing the size of each chunk. Default is 0.
+        - fn: A function that takes the current index and the iterable as arguments and returns the size of the chunk. Default is None.
+
+        Returns:
+        An iterator that yields chunks of the input iterable.
+
+        Example usage:
+        ```
+        data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        for chunk in chunks(data, 3):
+            print(chunk)
+        ```
+        Output:
+        ```
+        [1, 2, 3]
+        [4, 5, 6]
+        [7, 8, 9]
+        [10]
+        ```
+        """
+        arr = []
+        _iter = tuple(_iter)
+        for i, x in enumerate(_iter):
+            arr.append(x)
+            if len(arr) == (fn(i, _iter) if fn else n):
+                yield arr
+                arr = []
+
+        if arr:
+            yield arr
+
+
+def configure_pad_token(
+    tokenizer: "PreTrainedTokenizerBase",
+    model_config: Optional["PretrainedConfig"] = None,
+) -> "PreTrainedTokenizerBase":
+    """
+    This function checks if the (Hugging Face) tokenizer has a padding token and sets it if not present.
+    Some tokenizers require special handling.
+
+    Args:
+        tokenizer: The tokenizer for which the padding token is to be handled.
+        model_config: The configuration of the model. Default is None.
+
+    Returns:
+        The tokenizer after the padding token has been handled.
+
+    Raises:
+        AssertionError: If the tokenizer is of type RWKVWorldTokenizer or Rwkv5Tokenizer and the padding token id is not 0.
+    """
+    if tokenizer.pad_token:
+        pass
+    elif tokenizer.unk_token:
+        tokenizer.pad_token_id = tokenizer.unk_token_id
+    elif tokenizer.eos_token:
+        tokenizer.pad_token_id = tokenizer.eos_token_id
+    else:
+        # handle special cases
+        if model_config and getattr(model_config, "model_type", None) == "qwen":
+            # Qwen's trust_remote_code tokenizer does not allow for adding special tokens
+            tokenizer.pad_token = "<|endoftext|>"
+        elif (
+            tokenizer.__class__.__name__ == "RWKVWorldTokenizer"
+            or tokenizer.__class__.__name__ == "Rwkv5Tokenizer"
+        ):
+            # The RWKV world tokenizer, does not allow for adding special tokens / setting the pad token (which is set as 0)
+            # The additional tokenizer name check is needed, as there exists rwkv4 models with neox tokenizer
+            # ---
+            # Note that the world tokenizer class name, might change in the future for the final huggingface merge
+            # https://github.com/huggingface/transformers/pull/26963
+            assert tokenizer.pad_token_id == 0
+        else:
+            tokenizer.add_special_tokens({"pad_token": "<|pad|>"})
+
+    return tokenizer
+
+
+def replace_placeholders(
+    string: str, default_placeholder: str, image_token: str, max_images: int
+):
+    """
+    A utility function used for local multimodal models. It locates all `placeholder` string
+    occurrences in the given input `string_` and replaces the first `max_count` instances with
+    `replacement`, and all subsequent occurrences with the empty string.
+
+    This is used to replace <image> placeholder tags by model-specific image tokens like <|image_pad|>
+    and to allow for only the first `max_count` images to be passed to a model if desired.
+
+    :param string: The original string containing placeholders.
+    :param default_placeholder: The placeholder text to be replaced.
+    :param image_token: The token to replace the placeholder with.
+    :param max_images: The maximum number of replacements to make.
+    :return: The string with placeholders replaced.
+    """
+    count = 0
+    result = []
+
+    parts = string.split(default_placeholder)
+    for part in parts[:-1]:  # Iterate through all but the last part
+        result.append(part)
+        if count < max_images:
+            result.append(image_token)
+            count += 1
+        elif default_placeholder != image_token:
+            result.append(default_placeholder)
+
+    # Add the last part of the string
+    result.append(parts[-1])
+    return "".join(result)
+
+
+def flatten_image_list(images: List[List]):
+    """
+    Takes in a list of lists of images, and returns a single list of all images in order.
+    Used for some multimodal models like Llava-1.5 which expects this flattened-list format for its image processor.
+
+    :param images: A list of lists of PIL images.
+    :return: a list of PIL images, via concatenating all the sub-lists in order.
+    """
+    return [image for image_list in images for image in image_list]
+
+
+def handle_stop_sequences(
+    until: Union[str, List[str], None], eos: Optional[str]
+) -> List[str]:
+    """Ensures that the `until` parameter is a list of stop sequences and includes the EOS token."""
+    if isinstance(until, str):
+        until = [until]
+    elif until is None:
+        until = []
+    elif not isinstance(until, list):
+        raise ValueError(
+            f"Expected `kwargs['until']` to be of type Union[str,list] but got {until}"
+        )
+
+    if eos is not None and eos not in until:
+        until.append(eos)
+    return until
+
+
+def resize_image(
+    image: "Image.Image",
+    width: Optional[int] = None,
+    height: Optional[int] = None,
+    max_dimension: Optional[int] = None,
+    keep_aspect_ratio: bool = True,
+    resample_filter: Union[int, str] = "Image.BICUBIC",
+    min_width: int = 1,
+    min_height: int = 1,
+) -> "Image.Image":
+    """
+    Resizes a PIL Image object with flexible options.
+
+    Args:
+        image: The PIL Image object to resize.
+        width: Target width in pixels.
+        height: Target height in pixels.
+        max_dimension: Maximum size for the longer dimension of the image.
+        keep_aspect_ratio: If True (default) and both width and height are provided,
+                          the image is resized to fit within these dimensions while
+                          maintaining its aspect ratio. If False, the image is stretched
+                          to the exact width and height.
+        resample_filter: The resampling filter to use for resizing.
+                        Defaults to Image.BICUBIC.
+        min_width: Minimum width for the resized image. Defaults to 1.
+        min_height: Minimum height for the resized image. Defaults to 1.
+
+    Returns:
+        The resized PIL Image object. If no resize parameters are provided
+        or if the image already meets the criteria, the original image is returned.
+
+    Order of precedence for resizing:
+    1. If width AND height are provided:
+       - If keep_aspect_ratio is True: Fits image within bounds, preserving aspect ratio.
+       - If keep_aspect_ratio is False: Resizes to exact dimensions (may distort).
+    2. Else if only width is provided: Calculates height proportionally.
+    3. Else if only height is provided: Calculates width proportionally.
+    4. Else if max_dimension is provided: Resizes the longest side to max_dimension
+       and scales the other side proportionally.
+    5. If none of the above are provided, returns the original image.
+    """
+    original_width, original_height = image.size
+
+    # If no arguments are provided, return the original image
+    if width is None and height is None and max_dimension is None:
+        return image
+
+    new_width = original_width
+    new_height = original_height
+
+    if width is not None and height is not None:
+        # No resize needed if image is already smaller than target dimensions
+        if original_width <= width and original_height <= height:
+            return image
+
+        if keep_aspect_ratio:
+            # Calculate the ratio to fit within the target dimensions
+            ratio = min(width / original_width, height / original_height)
+            new_width = int(original_width * ratio)
+            new_height = int(original_height * ratio)
+        else:
+            # Stretch to exact dimensions
+            new_width = width
+            new_height = height
+    elif width is not None:
+        # No resize needed if width is already smaller
+        if original_width <= width:
+            return image
+        # Calculate height proportionally
+        new_width = width
+        new_height = int((original_height / original_width) * new_width)
+    elif height is not None:
+        # No resize needed if height is already smaller
+        if original_height <= height:
+            return image
+        # Calculate width proportionally
+        new_height = height
+        new_width = int((original_width / original_height) * new_height)
+    elif max_dimension is not None:
+        # No resize needed if both dimensions are smaller than max_dimension
+        if max(original_height, original_width) <= max_dimension:
+            return image
+
+        if original_width > original_height:
+            # Width is the longer side
+            new_width = max_dimension
+            new_height = int((original_height / original_width) * new_width)
+        else:
+            # Height is the longer side or sides are equal
+            new_height = max_dimension
+            new_width = int((original_width / original_height) * new_height)
+
+    # Ensure dimensions are at least minimum values
+    new_width = max(min_width, new_width)
+    new_height = max(min_height, new_height)
+
+    # Perform the resize operation with the calculated dimensions
+    return image.resize((new_width, new_height), resample_filter)
+
+
+def truncate_tokens(
+    tokens: List[int],
+    max_length: int,
+    tokenizer: "PreTrainedTokenizerBase",
+    strategy: str = "left",
+):
+    if strategy == "left":
+        return tokens[-max_length:]
+    elif strategy == "right":
+        return tokens[:max_length]
+    elif strategy == "middle":
+        # Truncate the middle of the sequence
+        left_length = max_length // 2
+        right_length = max_length - left_length
+        return tokens[:left_length] + tokens[-right_length:]
+    return None

lm-evaluation-harness/lm_eval/models/vllm_vlms.py

@@ -0,0 +1,323 @@
+import copy
+import logging
+from typing import Dict, List, Optional
+
+import transformers
+from more_itertools import distribute
+from tqdm import tqdm
+
+from lm_eval.api.instance import Instance
+from lm_eval.api.registry import register_model
+from lm_eval.models.utils import (
+    Collator,
+    handle_stop_sequences,
+    replace_placeholders,
+    resize_image,
+    undistribute,
+)
+from lm_eval.models.vllm_causallms import VLLM
+
+
+eval_logger = logging.getLogger(__name__)
+
+
+try:
+    import ray
+    from vllm import LLM, SamplingParams
+    from vllm.lora.request import LoRARequest  # noqa: F401
+    from vllm.transformers_utils.tokenizer import get_tokenizer  # noqa: F401
+except ModuleNotFoundError:
+    pass
+
+
+DEFAULT_IMAGE_PLACEHOLDER = "<image>"
+
+
+@register_model("vllm-vlm")
+class VLLM_VLM(VLLM):
+    MULTIMODAL = True
+
+    def __init__(
+        self,
+        pretrained: str,
+        trust_remote_code: Optional[bool] = False,
+        revision: Optional[str] = None,
+        interleave: bool = True,
+        # TODO<baber>: handle max_images and limit_mm_per_prompt better
+        max_images: int = 999,
+        image_width: Optional[int] = None,
+        image_height: Optional[int] = None,
+        image_max_side: Optional[int] = None,
+        **kwargs,
+    ):
+        self.image_width = image_width
+        self.image_height = image_height
+        self.image_max_side = image_max_side
+        if self.image_max_side and (self.image_width or self.image_height):
+            raise ValueError(
+                "Ambiguous config for image resize: you can not specify both "
+                "image_max_side and (image_width or image_height)"
+            )
+
+        if max_images != 999:
+            kwargs["limit_mm_per_prompt"] = {"image": max_images}
+            eval_logger.info(f"Setting limit_mm_per_prompt[image] to {max_images}")
+        super().__init__(
+            pretrained=pretrained,
+            trust_remote_code=trust_remote_code,
+            revision=revision,
+            **kwargs,
+        )
+        self.interleave = interleave
+        self.max_images = max_images
+        self.processor = transformers.AutoProcessor.from_pretrained(
+            pretrained,
+            revision=revision,
+            trust_remote_code=trust_remote_code,
+        )
+        self.chat_applied: bool = False
+
+    def tok_batch_multimodal_encode(
+        self,
+        strings: List[str],  # note that input signature of this fn is different
+        images,  # TODO: typehint on this
+        left_truncate_len: int = None,
+        truncation: bool = False,
+    ):
+        images = [img[: self.max_images] for img in images]
+        # TODO<baber>: is the default placeholder always <image>?
+        if self.chat_applied is False:
+            strings = [
+                replace_placeholders(
+                    string,
+                    DEFAULT_IMAGE_PLACEHOLDER,
+                    DEFAULT_IMAGE_PLACEHOLDER,
+                    self.max_images,
+                )
+                for string in strings
+            ]
+
+        outputs = []
+        for x, i in zip(strings, images):
+            inputs = {
+                "prompt": x,
+                "multi_modal_data": {"image": i},
+            }
+            outputs.append(inputs)
+        return outputs
+
+    def _multimodal_model_generate(
+        self,
+        requests: List[List[dict]] = None,
+        generate: bool = False,
+        max_tokens: int = None,
+        stop: Optional[List[str]] = None,
+        **kwargs,
+    ):
+        if generate:
+            kwargs = self.modify_gen_kwargs(kwargs)
+            sampling_params = SamplingParams(max_tokens=max_tokens, stop=stop, **kwargs)
+        else:
+            sampling_params = SamplingParams(
+                temperature=0, prompt_logprobs=1, max_tokens=1, detokenize=False
+            )
+        if self.data_parallel_size > 1:
+            # vLLM hangs if resources are set in ray.remote
+            # also seems to only work with decorator and not with ray.remote() fn
+            # see https://github.com/vllm-project/vllm/issues/973
+            @ray.remote
+            def run_inference_one_model(
+                model_args: dict, sampling_params, requests: List[List[dict]]
+            ):
+                llm = LLM(**model_args)
+                return llm.generate(requests, sampling_params=sampling_params)
+
+            # dispatch requests to all self.data_parallel_size workers, in interleaved fashion
+            # interleaved important to balance context lengths across workers
+            requests = [list(x) for x in distribute(self.data_parallel_size, requests)]
+            inputs = ((self.model_args, sampling_params, req) for req in requests)
+            object_refs = [run_inference_one_model.remote(*x) for x in inputs]
+            results = ray.get(object_refs)
+            # Invoke ray.shutdown() to prevent hang-ups if subsequent calls required.
+            ray.shutdown()
+            # flatten results
+            return undistribute(results)
+
+        if self.lora_request is not None:
+            outputs = self.model.generate(
+                requests,
+                sampling_params=sampling_params,
+                use_tqdm=True if self.batch_size == "auto" else False,
+                lora_request=self.lora_request,
+            )
+        else:
+            outputs = self.model.generate(
+                requests,
+                sampling_params=sampling_params,
+                use_tqdm=True if self.batch_size == "auto" else False,
+            )
+        return outputs
+
+    def apply_chat_template(
+        self, chat_history: List[Dict[str, str]], add_generation_prompt=True
+    ) -> str:
+        self.chat_applied = True
+        if not self.interleave:
+            for content in chat_history:
+                c = []
+                text = content["content"]
+
+                # Count and remove image placeholders
+                image_count = min(
+                    self.max_images, text.count(DEFAULT_IMAGE_PLACEHOLDER)
+                )
+                text = text.replace(DEFAULT_IMAGE_PLACEHOLDER, "")
+
+                # Add image entries
+                for _ in range(image_count):
+                    c.append({"type": "image", "image": None})
+
+                # Add single text entry at the end
+                c.append({"type": "text", "text": text})
+
+                content["content"] = c
+        else:
+            for content in chat_history:
+                c = []
+                text = content["content"]
+                expected_image_count = min(
+                    self.max_images, text.count(DEFAULT_IMAGE_PLACEHOLDER)
+                )
+                actual_image_count = 0
+
+                text_parts = text.split(DEFAULT_IMAGE_PLACEHOLDER)
+
+                for i, part in enumerate(text_parts):
+                    # TODO: concatenate text parts (esp. if skipping images)?
+                    if part:  # Add non-empty text parts
+                        c.append({"type": "text", "text": part})
+                    if (
+                        (i < len(text_parts) - 1) and i < self.max_images
+                    ):  # Add image placeholder after each split except the last
+                        c.append({"type": "image"})
+                        actual_image_count += 1
+
+                content["content"] = c
+
+                if actual_image_count != expected_image_count:
+                    raise ValueError(
+                        f"Mismatch in image placeholder count. Expected: {expected_image_count}, Actual: {actual_image_count}"
+                    )
+
+        return self.processor.apply_chat_template(
+            chat_history,
+            add_generation_prompt=add_generation_prompt,
+            continue_final_message=not add_generation_prompt,
+        )
+
+    def generate_until(
+        self, requests: List[Instance], disable_tqdm: bool = False
+    ) -> List[str]:
+        if requests and len(requests[0].args) < 3:
+            # Fall back to non-multimodal generation.
+            return super().generate_until(requests=requests, disable_tqdm=disable_tqdm)
+
+        res = []
+
+        def _collate(x):
+            # the negative sign on len(toks) sorts descending - this has a few advantages:
+            # - time estimates will always be over not underestimates, which is more useful for planning
+            # - to know the size of a batch when going through the list, you know the first one is always the batch
+            #   padded context length. this is useful to simplify the batching logic and more importantly to make
+            #   automatic adaptive batches much much easier to implement
+            # - any OOMs will happen right away rather than near the end
+            toks = self.tok_encode(x[0])
+            return -len(toks), x[0]
+
+        pbar = tqdm(
+            total=len(requests),
+            disable=(disable_tqdm or (self.rank != 0)),
+            desc="Running generate_until requests with text+image input",
+        )
+        # TODO: port auto-batch sizing into this.
+
+        # we group requests by their generation_kwargs,
+        # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
+        # in the same batch.
+        re_ords = Collator(
+            [reg.args for reg in requests],
+            _collate,
+            group_by="gen_kwargs",
+            group_fn=lambda x: x[1],
+        )
+        chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
+        eos = self.tokenizer.decode(self.eot_token_id)
+        for chunk in chunks:
+            contexts, all_gen_kwargs, aux_arguments = zip(*chunk)
+
+            visuals = [
+                [
+                    resize_image(
+                        img, self.image_width, self.image_height, self.image_max_side
+                    )
+                    for img in arg["visual"]
+                ]
+                for arg in aux_arguments
+            ]
+
+            if not isinstance(contexts, list):
+                contexts = list(
+                    contexts
+                )  # for Qwen2-VL, processor is unhappy accepting a tuple of strings instead of a list.
+                # TODO: could we upstream this workaround to HF?
+
+            # we assume all gen kwargs in the batch are the same
+            # this is safe to assume because the `grouper` object ensures it.
+            gen_kwargs = all_gen_kwargs[0]
+            # unpack our keyword arguments.
+            if isinstance(gen_kwargs, dict):
+                kwargs = copy.deepcopy(gen_kwargs)  # edge case for repeats > 1
+                # add EOS token to stop sequences
+                until = handle_stop_sequences(kwargs.pop("until", None), eos=eos)
+            else:
+                raise ValueError(
+                    f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}"
+                )
+            if "max_gen_toks" in kwargs.keys():
+                max_gen_toks = kwargs.pop("max_gen_toks")
+            else:
+                max_gen_toks = self.max_gen_toks
+
+            max_ctx_len = self.max_length - max_gen_toks
+
+            inputs = self.tok_batch_multimodal_encode(
+                contexts,
+                visuals,
+                left_truncate_len=max_ctx_len,
+            )
+
+            cont = self._multimodal_model_generate(
+                inputs, stop=until, generate=True, max_tokens=max_gen_toks, **kwargs
+            )
+
+            for output, context in zip(cont, contexts):
+                generated_text = output.outputs[0].text
+                res.append(generated_text)
+                self.cache_hook.add_partial(
+                    "generate_until", (context, gen_kwargs), generated_text
+                )
+                pbar.update(1)
+        # reorder this group of results back to original unsorted form
+        res = re_ords.get_original(res)
+
+        pbar.close()
+        return res
+
+    def loglikelihood_rolling(self, requests: List[Instance]) -> List[float]:
+        if requests and len(requests[0].args) < 3:
+            # Fall back to non-multimodal generation.
+            return super().loglikelihood_rolling(requests=requests)
+        raise NotImplementedError(
+            "model type `vllm-vlm` does not support loglikelihood_rolling. Use 'vlm' model type for text-only loglikelihood_rolling tasks ",
+            "this is because we do not support measuring the loglikelihood a model assigns to an image.",
+        )

lm-evaluation-harness/lm_eval/tasks/aclue/README.md

@@ -0,0 +1,50 @@
+# ACLUE
+
+### Paper
+
+Can Large Language Model Comprehend Ancient Chinese? A Preliminary Test on ACLUE
+https://arxiv.org/abs/2310.09550
+
+The Ancient Chinese Language Understanding Evaluation (ACLUE) is an evaluation benchmark focused on ancient Chinese language comprehension. It aims to assess the performance of large-scale language models on understanding ancient Chinese. The benchmark comprises 15 tasks spanning various domains, including lexical, syntactic, semantic, inference, and knowledge. ACLUE's tasks are derived from a combination of manually curated questions from publicly available resources, and automatically
+generated questions from classical Chinese language corpora. The range of questions span from the Xia dynasty (2070 BCE) to the Ming dynasty (1368 CE). ACLUE adopts a multiple-choice question format for all tasks.
+
+Homepage: https://github.com/isen-zhang/ACLUE
+
+### Citation
+
+```bibtex
+@inproceedings{zhang-li-2023-large,
+    title = "Can Large Language Model Comprehend {A}ncient {C}hinese? A Preliminary Test on {ACLUE}",
+    author = "Zhang, Yixuan  and Li, Haonan",
+    booktitle = "Proceedings of the Ancient Language Processing Workshop",
+    month = sep,
+    year = "2023",
+    address = "Varna, Bulgaria",
+    publisher = "INCOMA Ltd., Shoumen, Bulgaria",
+    url = "https://aclanthology.org/2023.alp-1.9",
+    pages = "80--87"
+}
+```
+
+### Groups, Tags, and Tasks
+
+#### Groups
+
+- `aclue`: All 15 subjects of the ACLUE dataset, evaluated following the methodology in CMMLU's original implementation.
+
+#### Tasks
+
+The following tasks evaluate subjects in the ACLUE dataset using loglikelihood-based multiple-choice scoring:
+- `aclue_{subject_english}`
+
+### Checklist
+
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation?
+    * [x] Yes, original implementation contributed by author of the benchmark
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?

lm-evaluation-harness/lm_eval/tasks/aclue/aclue_named_entity_recognition.yaml

@@ -0,0 +1,4 @@
+"dataset_name": "named_entity_recognition"
+"description": "以下是关于古汉语命名体识别的单项选择题，请直接给出正确答案的选项。\n\n"
+"include": "_default_template_yaml"
+"task": "aclue_named_entity_recognition"

lm-evaluation-harness/lm_eval/tasks/aclue/aclue_poetry_context_prediction.yaml

@@ -0,0 +1,4 @@
+"dataset_name": "poetry_context_prediction"
+"description": "以下是关于古诗词上下句预测的单项选择题，请直接给出正确答案的选项。\n\n"
+"include": "_default_template_yaml"
+"task": "aclue_poetry_context_prediction"

lm-evaluation-harness/lm_eval/tasks/aclue/aclue_reading_comprehension.yaml

@@ -0,0 +1,4 @@
+"dataset_name": "reading_comprehension"
+"description": "以下是关于古文阅读理解的单项选择题，请直接给出正确答案的选项。\n\n"
+"include": "_default_template_yaml"
+"task": "aclue_reading_comprehension"

lm-evaluation-harness/lm_eval/tasks/aclue/aclue_sentence_segmentation.yaml

@@ -0,0 +1,4 @@
+"dataset_name": "sentence_segmentation"
+"description": "以下是关于古文断句的单项选择题，请直接给出正确答案的选项。\n\n"
+"include": "_default_template_yaml"
+"task": "aclue_sentence_segmentation"

lm-evaluation-harness/lm_eval/tasks/acpbench/README.md

@@ -0,0 +1,130 @@
+# ACPBench
+
+**Homepage:** https://ibm.github.io/ACPBench/
+
+### Papers
+
+**Title:** ACPBench: Reasoning About Action, Change, and Planning
+**Pdf:** https://arxiv.org/pdf/2410.05669
+**Task:** `acp_bench`
+**Abstract:**
+
+There is an increasing body of work using Large Language Models (LLMs) as agents for orchestrating workflows and making decisions in domains that require planning and multi-step reasoning. As a result, it is imperative to evaluate LMs on core skills required for planning. ACPBench is a benchmark for evaluating the reasoning tasks in the field of planning. The benchmark consists of 7 reasoning tasks over 13 planning domains. The collection is constructed from planning domains described in a formal language. This allows the synthesized problems to have provably correct solutions across many tasks and domains. Further, it allows the luxury to scale without additional human effort, i.e., many additional problems can be created automatically.
+
+
+
+**Title:** ACPBench Hard: Unrestrained Reasoning about Action, Change, and Planning
+**Pdf:** https://arxiv.org/abs/2503.24378
+**Task:** `acp_bench_hard`
+**Abstract:**
+
+We introduce ACPBench Hard, a dataset of generative, open-ended questions which LLM models needs to answer in order to plan. Models that perform well on these tasks could in principle be integrated into a planner or be used directly as a policy. We discuss the complexity of these tasks as well as the complexity of validating the correctness of their answers and present validation algorithms for each task. Equipped with these validators, we test the performance of a variety of models on our tasks and find that for most of these tasks, the performance of even the largest models is still subpar. Our experiments show that no model outperforms any other in these tasks, and with a few exceptions, all tested language models score below 65\%, indicating that even the current frontier language models as well as so-called reasoning models have a long way to go before they can reliably reason about planning.
+
+The dataset is available on [HuggingFace](https://huggingface.co/datasets/ibm-research/acp_bench).
+
+
+### Citation
+
+```
+@inproceedings{kokel2025acp
+  author       = {Harsha Kokel and
+                  Michael Katz and
+                  Kavitha Srinivas and
+                  Shirin Sohrabi},
+  title        = {ACPBench: Reasoning about Action, Change, and Planning},
+  booktitle    = {{AAAI}},
+  publisher    = {{AAAI} Press},
+  year         = {2025}
+}
+
+@misc{KokelKSS25ACPHard,
+  title       = {ACPBench Hard: Unrestrained Reasoning about Action, Change, and Planning},
+  author      = {Harsha Kokel and
+                 Michael Katz and
+                 Kavitha Srinivas and
+                 Shirin Sohrabi},
+  year        = {2025},
+  eprint      = {2503.24378},
+  archivePrefix = {arXiv},
+  primaryClass  = {cs.AI},
+  url         = {https://arxiv.org/abs/2503.24378},
+}
+```
+
+### Groups, Tags, and Tasks
+
+#### Groups
+
+* None
+
+#### Tags
+
+* `acp_bench` : Evaluates `acp_bool_cot_2shot` and `acp_mcq_cot_2shot` (Main variant for ACPBench paper)
+* `acp_bool_cot_2shot` : Evaluates `acp_areach_bool`, `acp_app_bool`, `acp_just_bool`, `acp_land_bool`, `acp_prog_bool`, `acp_reach_bool`, `acp_val_bool` with chain-of-thought and 2 shots
+* `acp_mcq_cot_2shot` : Evaluates `acp_areach_mcq`, `acp_app_mcq`, `acp_just_mcq`, `acp_land_mcq`, `acp_prog_mcq`, `acp_reach_mcq`, `acp_val_mcq`  with chain-of-thought and 2 shots
+* `acp_bench_hard` : Evaluates `acp_gen_2shot` (Main variant for ACPBench Hard paper)
+* `acp_gen_2shot` : Evaluates `acp_areach_gen`, `acp_app_gen`, `acp_just_gen`, `acp_land_gen`, `acp_nexta_gen`, `acp_prog_gen`, `acp_reach_gen`, `acp_val_gen` with 2 shots
+* `acp_bench_hard_with_pddl` : Evaluates `acp_gen_2shot_with_pddl`
+* `acp_gen_2shot_with_pddl` : Evaluates `acp_areach_gen_with_pddl`, `acp_app_gen_with_pddl`, `acp_just_gen_with_pddl`, `acp_land_gen_with_pddl`, `acp_nexta_gen_with_pddl`, `acp_prog_gen_with_pddl`, `acp_reach_gen_with_pddl`, `acp_val_gen_with_pddl` with 2 shots
+
+#### Tasks
+
+7 Boolean tasks
+* `acp_areach_bool`
+* `acp_app_bool`
+* `acp_just_bool`
+* `acp_land_bool`
+* `acp_prog_bool`
+* `acp_reach_bool`
+* `acp_val_bool`
+
+7 MCQ tasks
+* `acp_areach_mcq`
+* `acp_app_mcq`
+* `acp_just_mcq`
+* `acp_land_mcq`
+* `acp_prog_mcq`
+* `acp_reach_mcq`
+* `acp_val_mcq`
+
+8 Generative tasks (with just natural language description in context)
+* `acp_areach_gen`
+* `acp_app_gen`
+* `acp_just_gen`
+* `acp_land_gen`
+* `acp_nexta_gen`
+* `acp_prog_gen`
+* `acp_reach_gen`
+* `acp_val_gen`
+
+and the same 8 generative tasks with natural language as well as the PDDL description of the domain and problem in context.
+* `acp_areach_gen_with_pddl`
+* `acp_app_gen_with_pddl`
+* `acp_just_gen_with_pddl`
+* `acp_land_gen_with_pddl`
+* `acp_nexta_gen_with_pddl`
+* `acp_prog_gen_with_pddl`
+* `acp_reach_gen_with_pddl`
+* `acp_val_gen_with_pddl`
+
+> ! The evaluation scripts are taken from original github https://github.com/IBM/ACPBench
+
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?
+
+
+### Change Log
+
+* 03/17/2025 Initial Commit
+* 05/13/2025 Adding ACPBench Hard tasks (with and without PDDL)

lm-evaluation-harness/lm_eval/tasks/acpbench/boolq_cot_2shot/_boolq_cot_2shot_yaml

@@ -0,0 +1,32 @@
+tag:
+  - acp_bool_cot_2shot
+  - acp_bench
+output_type: generate_until
+dataset_path: ibm-research/acp_bench
+test_split: test
+num_fewshot: 2
+doc_to_target: "{{answer}}"
+doc_to_text: "**Question**: {{context}}  {{question}} **Thoughts**:"
+filter_list:
+  - name: "extract-yes-no"
+    filter:
+      - function: "regex"
+        regex_pattern: "((?<=The answer is )(.*)(?=.)|(?<=the answer is )(.*)(?=.)|(?<=The answer: )(.*)(?=.)|(?<=The final answer: )(.*)(?=.)|(?<=..Final Answer..: )(.*)(?=.)|(?<=..answer..: )(.*)(?=.)|(?<=..Answer..: )(.*)(?=.)|\\b(Yes|No|yes|no)\\b)"
+      - function: "take_first"
+generation_kwargs:
+  until:
+    - "**Question**:"
+    - "**Question:**"
+    - "Q:"
+    - "\n\n\n\n"
+  do_sample: false
+  temperature: 0.0
+  max_gen_toks: 1024
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/acpbench/boolq_cot_2shot/app.yaml

@@ -0,0 +1,12 @@
+task: acp_app_bool
+dataset_name: acp_app_bool
+include: _boolq_cot_2shot_yaml
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: 'This is a ferry domain, where the task is to transport cars from their start to their goal locations, using a ferry. Each location is accessible by ferry from each other location. The cars can be debarked or boarded, and the ferry can carry only one car at a time.  There are 2 locations and 2 cars, numbered consecutively.  Currently, the ferry is at l0, with the car c1 on board. The cars are at locations as follows: c0 is at l0.'
+    question: 'Is the following action applicable in this state:  travel by sea from location l1 to location l0?'
+    answer: "Step 1: In order to apply the action travel by sea from location l1 to location l0, the following fact(s) must hold in this state: The ferry is at l1 location Step 2: These facts do not hold in the mentioned state: The ferry is at l1 location. So, the action is not applicable. **Final Answer**: No."
+  - context: 'There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations.  There are 3 trucks and 1 airplane, as well as 7 packages. There are 9 locations across 3 cities.  The locations are in cities as follows: l1-2, l1-0, and l1-1 are in c1; l2-0, l2-1, and l2-2 are in c2; l0-0, l0-2, and l0-1 are in c0.  Currently, p0 and p1 are at l1-2, a0 and p2 are at l0-0, t1 is at l1-0, t0 and p4 are at l0-2, t2 is at l2-0, p6, p3, and p5 are in t0.'
+    question:  'Is the following action applicable in this state:  load the object p4 from location l0-2 into the truck t0?'
+    answer: "Let's think step by step.  Step 1: In order to apply the action \"load the object p4 from location l0-2 into the truck t0\", the following fact(s) must hold in this state: p4 is at l0-2 and t0 is at l0-2 Step 2: These facts hold in the mentioned state, so the action is applicable. **Final Answer**: Yes."

lm-evaluation-harness/lm_eval/tasks/acpbench/boolq_cot_2shot/just.yaml

@@ -0,0 +1,13 @@
+task: acp_just_bool
+dataset_name: acp_just_bool
+include: _boolq_cot_2shot_yaml
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: 'This is a ferry domain, where the task is to transport cars from their start to their goal locations, using a ferry. Each location is accessible by ferry from each other location. The cars can be debarked or boarded, and the ferry can carry only one car at a time.  There are 2 locations and 2 cars, numbered consecutively.  Currently, the ferry is at l0 location and it is empty. The cars are at locations as follows: c0 and c1 are at l0. The goal is to reach a state where the following facts hold: Car c1 is at location l1 and Car c0 is at location l1'
+    question: 'Given the plan: \"board the car c1 at the location l0, sail from location l0 to location l1, unload the car c1 from the ferry to location l1, travel by sea from location l1 to location l0, board the car c0 at location l0, travel by sea from location l0 to location l1, unload the car c0 from the ferry to location l1, embark the car c1 at location l1 on to the ferry, debark car c1 to location l1 from the ferry\"; can the following pair of consecutive actions be removed from this plan and still have a valid plan:  embark the car c1 at location l1 on to the ferry and debark the car c1 to location l1 from the ferry?'
+    answer: "Let's think step by step.   Step 1: A valid plan is a sequence of actions that is executable from the current state and achieves the goal.  Step 2: Consecutive actions \"embark the car c1 at location l1 on to the ferry and debark the car c1 to location l1 from the ferry\" can be removed from the plan and still have a valid plan. **Final Answer**: Yes."
+  - context: 'There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations.  There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities.  The locations are in cities as follows: l1-0 and l1-1 are in c1; l0-1 and l0-0 are in
+  c0.  Currently, p3, p2, and p1 are at l1-0, p0 and t1 are at l1-1, a0 is at l0-0, t0 is at l0-1. The goal is to reach a state where the following facts hold: p0 is at l0-0, p2 is at l1-0, p3 is at l0-1, and p1 is at l1-0. Given the plan: \"load the object p0 from location l1-1 into the truck t1, navigate the truck t1 which is in location l1-1 in city c1 to another location l1-0 in the same city, offload the object p0 from the truck t1 at location l1-0, drive truck t1 from location l1-0 in city c1 to location l1-1 in the same city, fly airplane a0 from airport l0-0 to airport l1-0, load the object p0 from location l1-0 onto the airplane a0, load the object p3 from location l1-0 onto the airplane a0, fly the airplane a0 from airport l1-0 to airport l0-0, unload the object p0 from the airplane a0 at location l0-0, remove the object p3 from the airplane a0 and place it on the location l0-0, drive the truck t0 in city c0 from location l0-1 to location l0-0, place the object p3 into the truck t0 at location l0-0, navigate the truck t0 from location l0-0 in city c0 to location l0-1 in the same city, unload object p3 from truck t0 at location l0-1\"'
+    question: 'can the following pair of consecutive actions be removed from this plan and still have a valid plan:  place the object p0 onto the airplane a0 at location l1-0 and place the object p3 onto the airplane a0 at location l1-0?'
+    answer: "Let's think step by step.   Step 1: A valid plan is a sequence of actions that is exectuable from the current state and achieves the goal.  Step 2: Consecutive actions \"place the object p0 onto the airplane a0 at location l1-0 and place the object p3 onto the airplane a0 at location l1-0\" cannot be removed from the plan and still have a valid plan. **Final Answer**: No."

lm-evaluation-harness/lm_eval/tasks/acpbench/boolq_cot_2shot/land.yaml

@@ -0,0 +1,13 @@
+task: acp_land_bool
+dataset_name: acp_land_bool
+include: _boolq_cot_2shot_yaml
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: 'This is a ferry domain, where the task is to transport cars from their start to their goal locations, using a ferry. Each location is accessible by ferry from each other location. The cars can be debarked or boarded, and the ferry can carry only one car at a time.  There are 2 locations and 5 cars, numbered consecutively.  Currently, the ferry is at l1 location and it is empty. The cars are at locations as follows: c4 and c2 are at l1; c0, c1, and c3 are at l0. The goal is to reach a state where the following facts hold: Car c4 is at location l0, Car c0 is at location l1, Car c1 is at location l0, Car c3 is at location l0, and Car c2 is at location l1.'
+    question: 'Is the following fact a landmark (must hold at some point along any plan) for the current state? The ferry is at l0 location.'
+    answer: "Let's think step by step.   Step 1: A fact is a landmark if it must hold at some point along any plan. Step 2: The fact \"The ferry is at l0 location.\" can be found by a simple procedure that traces back such atoms from the goal. **Final Answer**: Yes."
+  - context: 'There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations.  There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities.  The locations are in cities as follows: l1-1 and l1-0 are in c1; l0-0 and l0-1 are in c0.  Currently, t0 is at l0-0, t1, a0, p1, and p2 are at l1-0, p3 is in t1, p0 is in a0. The goal is to reach a state where the following facts hold: p2 is at l1-0, p0 is at l0-0, p3 is at l0-1, and p1 is at l1-0.'
+    question: 'Is the following fact a landmark (must hold at some point along any plan) for the current state? p2 is in a0.'
+    answer: "Let's think step by step.   Step 1: A fact is a landmark if it must hold at some point along any plan. Step 2: The fact \"p2 is in a0.\" is not achieved
+  along the following plan: remove the object p3 from the truck t1 and place it on the location l1-0, load the object p3 from location l1-0 into the airplane a0, load the object p2 from location l1-0 into the airplane a0, remove the object p2 from the airplane a0 and place it on the location l1-0, fly the airplane a0 from the airport l1-0 to the airport l0-0, unload the object p3 from the airplane a0 at location l0-0, remove the object p0 from the airplane a0 and place it on the location l0-0, place the object p3 into the truck t0 at location l0-0. drive the truck t0 in city c0 from location l0-0 to location l0-1, unload the object p3 from the truck t0 at location l0-1. **Final Answer**: No."

lm-evaluation-harness/lm_eval/tasks/acpbench/boolq_cot_2shot/val.yaml

@@ -0,0 +1,12 @@
+task: acp_val_bool
+dataset_name: acp_val_bool
+include: _boolq_cot_2shot_yaml
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: 'This is a ferry domain, where the task is to transport cars from their start to their goal locations, using a ferry. Each location is accessible by ferry from each other location. The cars can be debarked or boarded, and the ferry can carry only one car at a time.  There are 2 locations and 2 cars, numbered consecutively.  Currently, the ferry is at l0 location and it is empty. The cars are at locations as follows: c0 and c1 are at l0. The goal is to reach a state where the following facts hold: Car c1 is at location l1 and Car c0 is at location l1.'
+    question: 'Is the following sequence of actions \"embark the car c0 at location l0 on to the ferry travel by sea from location l0 to location l1 debark the car c0 to location l1 from the ferry sail from location l1 to location l0 board car c1 at location l0 travel by sea from location l0 to location l1 sail from location l1 to location l0\" a plan for the current state?'
+    answer: "Let's think step by step.   Step 1: For a sequence of actions to be a plan, it should be applicable in the current state, all actions should be valid, and the resulting state should entail the goal. Step 2: All the actions in the sequence are valid in this problem. Step 3: The action sequence is applicable in the current state. Step 4: However, the following goal fact(s) are not achieved: Car c1 is at location l1. **Final Answer**: No."
+  - context: 'This is a blocksworld domain where blocks can be placed on top of each other or on the table. There is one robotic arm that can move the block.  There are 3 blocks.  Currently, the robotic arm is empty. The following block(s) are on the table: block_1 and block_2. The following block(s) is stacked on top of another block: block_3 is on block_2. The goal is to reach a state where the following facts hold: The block block_1 is currently situated above the block block_2 and The block block_3 is on top of block block_1.'
+    question: 'Is the following sequence of actions "unstack the object block_3 from the object block_2 place the object block_3 on top of the object block_2 unstack object block_3 from object block_2 put down object block_3 collect the object block_3 from the table place the object block_3 on the table pick up object block_2 from the table place the object block_2 on the table pick up the object block_1 from the table stack the object block_1 on top of the object block_2 pick up object block_3 from the table stack object block_3 on top of object block_1" a plan for the current state? '
+    answer: "Let's think step by step.  Step 1: For a sequence of actions to be a plan, it should be applicable in the current state, all actions should be valid, and the resulting state should entail the goal. Step 2: All the actions in the sequence are valid in this problem. Step 3: The action sequence is applicable in the current state. Step 4: The action sequence achieves the goal. **Final Answer**: Yes."

lm-evaluation-harness/lm_eval/tasks/acpbench/gen_2shot/_gen_yaml_2shot

@@ -0,0 +1,25 @@
+tag:
+  - acp_gen_2shot
+  - acp_bench_hard
+dataset_path: ibm-research/acp_bench
+test_split: test
+doc_to_target: "{{answer}}"
+output_type: generate_until
+num_fewshot: 2
+generation_kwargs:
+  until:
+    - "\n\n\n\n"
+    - "\n\n"
+    - "**Question**:"
+    - "**Question:**"
+    - "Q:"
+  do_sample: false
+  max_gen_toks: 1000
+  temperature: 0.0
+metadata:
+  version: 1.0
+process_results: !function acp_utils.process_acp_results
+metric_list:
+  - metric: "score"
+    aggregation: mean
+    higher_is_better: True

lm-evaluation-harness/lm_eval/tasks/acpbench/gen_2shot/acp_grammar.lark

@@ -0,0 +1,23 @@
+NAME: /[a-zA-Z][a-zA-Z0-9-_]*/
+LPAR : "("
+RPAR : ")"
+LSPAR: "["
+RSPAR: "]"
+COMMA: ","
+WS: /[ \n]/
+
+action_none : "None"
+
+action_name : LPAR NAME (WS NAME)* RPAR
+
+action_list : (action_name WS?)*
+
+prog_list :  action_name* (COMMA action_name)*
+
+progression_list : LSPAR prog_list RSPAR LSPAR prog_list RSPAR
+
+act : action_name | action_none
+
+index: /[0-9]+[0-9]*/
+
+start: action_list

lm-evaluation-harness/lm_eval/tasks/acpbench/gen_2shot/acp_utils.py

@@ -0,0 +1,1128 @@
+import json
+import os
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from pathlib import Path
+
+from lm_eval.api.registry import register_filter
+from lm_eval.filters.extraction import RegexFilter
+
+
+try:
+    import tempfile
+
+    import tarski
+    from kstar_planner import planners as kp
+    from lark import Lark
+    from lark.lexer import Token
+    from lark.visitors import Visitor
+    from pddl.core import Problem
+    from pddl.parser.domain import DomainParser
+    from pddl.parser.problem import ProblemParser
+    from tarski.grounding.common import StateVariableLite
+    from tarski.grounding.lp_grounding import LPGroundingStrategy
+    from tarski.io import PDDLReader
+    from tarski.io import fstrips as iofs
+    from tarski.syntax.formulas import is_atom
+    from tarski.syntax.transform.action_grounding import (
+        ground_schema_into_plain_operator_from_grounding,
+    )
+    from tarski.util import SymbolIndex
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        "`lark>=1.1.9`, `tarski[clingo]==0.8.2`, `pddl==0.4.2` and `kstar-planner==1.4.2` are required for evaluating the generative tasks. \
+Please install via pip install lm-eval[acpbench] or pip install -e .[acpbench]",
+    )
+
+
+#########################################################################
+# Grammar
+
+
+GRAMMAR_FILE = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "acp_grammar.lark"
+)
+
+
+class ACPBench_Visitor(Visitor):
+    def __init__(self) -> None:
+        super().__init__()
+        self.action_lists = None
+        self.action_names = None
+        self.progression_lists = None
+        self.prog_lists = None
+        self.indexes = None
+
+    def action_list(self, tree):
+        self.action_lists = []
+
+    def prog_list(self, tree):
+        if self.prog_lists is not None:
+            self.progression_lists.append(self.prog_lists)
+        self.prog_lists = []
+
+    def progression_list(self, tree):
+        self.progression_lists = []
+
+    def action_none(self, tree):
+        self.action_names = "None"
+
+    def action_name(self, tree):
+        act_name = "(" + "".join(tree.children[1:-1]) + ")"
+        self.action_names = act_name
+        if self.action_lists is not None:
+            self.action_lists.append(act_name)
+        if self.prog_lists is not None:
+            self.prog_lists.append(act_name)
+
+    def index(self, tree):
+        self.indexes = "".join(tree.children)
+        if not self.indexes.isnumeric():
+            self.indexes = None
+
+
+class ACPGrammarParser(object):
+    def __init__(self, task) -> None:
+        self.task = task
+        with open(GRAMMAR_FILE) as f:
+            grammar = f.read()
+            self.acp_parser = Lark(grammar, start=task, parser="lalr")
+
+    def parse(self, input, debug=False):
+        def ignore_errors(e):
+            if hasattr(e, "token") and e.token.type == "$END":
+                for x in e.expected:
+                    if x != "WS":
+                        e.interactive_parser.feed_token(
+                            Token(x, self.acp_parser.get_terminal(x).pattern.value)
+                        )
+
+            return True
+
+        input = input.replace("\n", "")
+        input = input.strip()
+        try:
+            tree = self.acp_parser.parse(input, on_error=ignore_errors)
+
+            if debug:
+                print(tree)
+            visitor = ACPBench_Visitor()
+            visitor.visit_topdown(tree)
+            if self.task == "action_list":
+                return visitor.action_lists
+            elif self.task == "act":
+                return visitor.action_names
+            elif self.task == "action_name":
+                return visitor.action_names
+            elif self.task == "index":
+                return visitor.indexes
+            elif self.task == "progression_list":
+                if visitor.prog_lists not in visitor.progression_lists:
+                    visitor.progression_lists.append(visitor.prog_lists)
+                return visitor.progression_lists
+        except Exception as e:
+            if debug:
+                print("exception")
+                print(e)
+            return None
+
+
+##############################################################################
+# Utils
+
+
+# Used in next action
+def is_on_optimal_plan(domain, problem, action, opt):
+    with (
+        tempfile.NamedTemporaryFile() as domain_temp,
+        tempfile.NamedTemporaryFile() as problem_temp,
+    ):
+        with open(str(domain_temp.name), "w", encoding="utf8") as file:
+            file.write(domain.lower())
+        with open(str(problem_temp.name), "w", encoding="utf8") as file:
+            file.write(problem.lower())
+
+        # Here, we need to keep the temp files live until the end of the function
+        try:
+            P = STRIPS(str(domain_temp.name), str(problem_temp.name))
+        except Exception:
+            # Unsolvable
+            return False
+
+        a = P.get_action_or_none(action[1:-1])
+        if a is None:
+            return False
+        state = P.init
+        next_state = progress(state, a)
+        if opt is None:
+            # Get an optimal plan cost
+            plans = generate_optimal_plans_for_problem_state(
+                P, state, num_plans=1, timeout=5
+            )
+            opt = len(plans[0]["actions"])
+        else:
+            opt = int(opt)
+
+        # Getting an optimal plan for the next state
+        next_plans = generate_optimal_plans_for_problem_state(
+            P, next_state, num_plans=1, timeout=5
+        )
+        if next_plans is None:
+            return False
+        next_opt = len(next_plans[0]["actions"])
+        return next_opt + 1 == opt
+
+
+# Used in justification
+def is_plan(domain, problem, new_plan):
+    P = get_STRIPS(domain, problem)
+    if P is None:
+        # Unsolvable
+        return False
+
+    # Check if new_plan is a plan
+    current_state = P.init
+    for action in new_plan:
+        applicable_actions = P.get_applicable_actions(current_state)
+        app_actions_list = [f"({a.name.lower()})" for a in applicable_actions]
+        if action.lower() not in app_actions_list:
+            return False
+        a = applicable_actions[app_actions_list.index(action.lower())]
+        current_state = progress(current_state, a)
+    return entails(current_state, P.goal)
+
+
+# Used in action reachability
+def get_action_preconditions(domain, problem, action):
+    P = get_STRIPS(domain, problem)
+
+    assert P is not None, f"Domain\n{domain}\nProblem\n{problem}\nAction: {action}"
+    a = P.get_action_or_none(action[1:-1])
+    if a is None:
+        return a
+
+    return [f"({f})" for f in a.pres]
+
+
+def generate_optimal_plans_for_problem_state(P, state, num_plans, timeout):
+    import tempfile
+
+    with (
+        tempfile.NamedTemporaryFile() as domain_temp,
+        tempfile.NamedTemporaryFile() as problem_temp,
+    ):
+        create_tmp_dom_prob_replace_init(P, state, domain_temp, problem_temp)
+        plans = generate_top_q_plans(
+            domain=str(domain_temp.name),
+            problem=str(problem_temp.name),
+            num_plans=num_plans,
+            quality_bound=1.0,
+            timeout=timeout,
+        )
+        # print(plans)
+        if plans is None or len(plans["plans"]) == 0:
+            return None
+        return plans["plans"]
+
+
+def generate_top_q_plans(domain, problem, num_plans=10, quality_bound=1.0, timeout=30):
+    # print("Running K* planner")
+    plans = kp.plan_unordered_topq(
+        domain_file=Path(domain),
+        problem_file=Path(problem),
+        number_of_plans_bound=num_plans,
+        quality_bound=quality_bound,
+        timeout=timeout,
+    )
+    return plans
+
+
+# Used in (action) reachability
+def is_unsolvable_new_goal(domain, problem, new_goal):
+    goal = extract_goal(problem)
+    new_problem = problem.replace(goal, f"(:goal {new_goal} )")
+    return is_unsolvable(domain, new_problem)
+
+
+def is_unsolvable(domain, problem):
+    with (
+        tempfile.NamedTemporaryFile() as domain_temp,
+        tempfile.NamedTemporaryFile() as problem_temp,
+    ):
+        with open(str(domain_temp.name), "w", encoding="utf8") as file:
+            file.write(str(domain))
+        with open(str(problem_temp.name), "w", encoding="utf8") as file:
+            file.write(str(problem))
+
+        plans = kp.plan_unordered_topq(
+            domain_file=Path(str(domain_temp.name)),
+            problem_file=Path(str(problem_temp.name)),
+            quality_bound=1.0,
+            number_of_plans_bound=1,
+            timeout=3,
+        )
+
+        if len(plans["planner_error"]) > 0:
+            fl = plans["planner_error"].split("\n")[0]
+            print(f"Planner error: {fl}")
+            return False
+        if plans is None or len(plans["plans"]) == 0:
+            return plans["unsolvable"]
+        return False
+
+
+def extract_goal(prob):
+    a = prob.split("(:goal")[1]
+    cp = 1
+    for i, c in enumerate(a):
+        if c == ")":
+            cp -= 1
+        if c == "(":
+            cp += 1
+        if cp == 0:
+            return "(:goal" + a[: i + 1]
+
+    assert False
+
+
+def entails(state, partialstate):
+    return partialstate <= state
+
+
+def progress(state, act):
+    assert entails(state, act.pres), (
+        "Cannot progress with inconsistent state / action precondition:\n\t Action: "
+        + act.name
+        + "\n\t State: \n\t\t"
+        + "\n\t\t".join(state)
+    )
+    return (state - act.dels) | act.adds
+
+
+def regress(state, act):
+    assert len(state & act.dels) == 0, (
+        "Cannot regress with inconsistent state / action delete effect:\n\t Action: "
+        + act.name
+        + "\n\t State: \n\t\t"
+        + "\n\t\t".join(state)
+    )
+    return (state - act.adds) | act.pres
+
+
+def get_STRIPS(domain, problem):
+    with (
+        tempfile.NamedTemporaryFile() as domain_temp,
+        tempfile.NamedTemporaryFile() as problem_temp,
+    ):
+        with open(str(domain_temp.name), "w", encoding="utf8") as file:
+            file.write(domain.lower())
+        with open(str(problem_temp.name), "w", encoding="utf8") as file:
+            file.write(problem.lower())
+
+        try:
+            P = STRIPS(str(domain_temp.name), str(problem_temp.name))
+            return P
+        except Exception as e:
+            print(f"||{e}||")
+            return None
+
+
+def create_tmp_dom_prob_replace_init(P, state, result_domain_file, result_problem_file):
+    d, p = P.PDDL_replace_init_pddl_parser(state)
+    with open(str(result_domain_file.name), "w", encoding="utf8") as file:
+        file.write(str(d))
+    with open(str(result_problem_file.name), "w", encoding="utf8") as file:
+        file.write(str(p))
+
+    return d, p
+
+
+def fix_name(s):
+    # (act param)
+    if "(" == s[0] and ")" == s[-1]:
+        return s[1:-1]
+    # make it space separated
+    s = s.replace(", ", " ").replace(",", " ")
+    # act(param)
+    if "(" in s:
+        assert ")" == s[-1], f"Broken name? {s}"
+        s = s.replace("(", " ").replace(")", "")
+    # act param
+    return s
+
+
+def get_atoms_pddl(d, p, atoms):
+    objs = set()
+    preds = defaultdict(list)
+    for atom in atoms:
+        a = atom.lower().strip().split(" ")
+        args = a[1:]
+        preds[a[0]].append(args)
+        objs |= set(args)
+
+    constants = [o for o in p.objects | d.constants if o.name.lower() in objs]
+    constants_dict = {}
+    for c in constants:
+        constants_dict[c.name.lower()] = c
+    assert len(objs) == len(constants), (
+        f"Could not identify all objects: {objs - set(constants_dict.keys())} not found, {set(constants_dict.keys()) - objs} should not be there"
+    )
+
+    state = []
+    covered_preds = set()
+    for f in d.predicates:
+        name = f.name.lower()
+        if name in preds:
+            covered_preds.add(name)
+            assert len(preds[name][0]) == f.arity, (
+                f"The arity does not match: {preds[name]} vs {f.terms}"
+            )
+            # Going over the lists of objects, adding ground predicate for each
+            for ob in preds[name]:
+                c = [constants_dict[o] for o in ob]
+                state.append(f(*c))
+    assert len(covered_preds) == len(preds.keys()), (
+        f"Covered predicates: \n{sorted(list(covered_preds))} vs \n{sorted(list(preds.keys()))}"
+    )
+    return set(state)
+
+
+class Action:
+    def __init__(self, name, pre, add, delete):
+        self.name = name
+        self.pres = pre
+        self.adds = add
+        self.dels = delete
+
+    def __str__(self):
+        pres = "{" + ", ".join([f"({a})" for a in self.pres]) + "}"
+        adds = "{" + ", ".join([f"({a})" for a in self.adds]) + "}"
+        dels = "{" + ", ".join([f"({a})" for a in self.dels]) + "}"
+
+        return f"< {self.name}, {pres}, {adds}, {dels} >"
+
+    def toJSON(self):
+        return json.dumps(
+            {
+                "name": self.name,
+                "preconditions": [f"({a})" for a in self.pres],
+                "add_effects": [f"({a})" for a in self.adds],
+                "delete_effects": [f"({a})" for a in self.dels],
+            },
+            sort_keys=True,
+            indent=4,
+        )
+
+    def __repr__(self):
+        return self.name
+
+    def __eq__(self, action):
+        return self.name == action.name
+
+    def __hash__(self):
+        return hash(self.name)
+
+
+class STRIPS:
+    def __init__(self, domain, problem):
+        self.domain_file = domain
+        self.problem_file = problem
+        self.reader = PDDLReader(raise_on_error=True)
+        self.reader.parse_domain(domain)
+        self.problem = self.reader.parse_instance(problem)
+        (self.grounded_fluents, init, goal, self.operators, self.grounder) = (
+            self.ground_problem(self.problem)
+        )
+
+        self.fluents = set([fix_name(str(f)) for f in self.grounded_fluents])
+        self.fluents_map = dict()
+        for f in self.grounded_fluents:
+            self.fluents_map[fix_name(str(f))] = f
+        self.init = set([fix_name(str(f)) for f in init])
+        self.goal = set([fix_name(str(f)) for f in goal])
+        self.actions = set()
+        self.action_map = {}
+        self.init_fluents = [self.fluents_map[f] for f in self.init]
+
+        self.static_predicates = [i.name for i in self.grounder.static_symbols]
+        for op in self.operators:
+            act = self.operator_to_action(op)
+            self.actions.add(act)
+            self.action_map[act.name.lower()] = act
+
+    def __str__(self):
+        fluents = "P = {" + ", ".join([f"({a})" for a in self.fluents]) + "}"
+        init = "I = {" + ", ".join([f"({a})" for a in self.init]) + "}"
+        goal = "G = {" + ", ".join([f"({a})" for a in self.goal]) + "}"
+        actions = "A = {" + "\n ".join([a.__str__() for a in self.actions]) + "}"
+        return fluents + ",\n" + init + "\n" + goal + "\n" + actions
+
+    def toJSON(self):
+        actions = [a.toJSON() for a in self.actions]
+        return json.dumps(
+            {
+                "fluents": list(self.fluents),
+                "initial_state": list(self.init),
+                "goal": list(self.goal),
+                "actions": actions,
+            },
+            sort_keys=True,
+            indent=4,
+        )
+
+    def operator_to_action(self, op, check_fluents=True, check_static=False):
+        adds = {
+            fix_name(str(f.atom)) for f in op.effects if isinstance(f, iofs.AddEffect)
+        } & self.fluents
+        dels = {
+            fix_name(str(f.atom)) for f in op.effects if isinstance(f, iofs.DelEffect)
+        } & self.fluents
+        pre = self.fix_pre_name(op.precondition)
+        if check_fluents:
+            pre = pre & self.fluents
+        if check_static:
+            pre = {p for p in pre if p.split()[0] not in self.static_predicates}
+        act = Action(fix_name(str(op)), pre, adds, dels)
+        return act
+
+    def fix_pre_name(self, precondition):
+        if not is_atom(precondition):
+            return {fix_name(str(f)) for f in precondition.subformulas}
+        return {fix_name(str(precondition))}
+
+    def action(self, name):
+        return self.action_map[fix_name(name).lower()]
+
+    def get_action_or_none(self, name):
+        if "(" in name and ")" != name[-1]:
+            return None
+        return self.action_map.get(fix_name(name).lower(), None)
+
+    def fluent(self, name):
+        return fix_name(name)
+
+    def static_symbols(self):
+        return list(self.grounder.static_symbols)
+
+    def fluent_symbols(self):
+        return list(self.grounder.fluent_symbols)
+
+    def get_grounded_atoms(self, symbol):
+        variables = SymbolIndex()
+        lang = symbol.language
+        key = "atom_" + symbol.name
+        model = self.grounder._solve_lp()
+        if (
+            key in model
+        ):  # in case there is no reachable ground state variable from that fluent symbol
+            for binding in model[key]:
+                binding_with_constants = tuple(lang.get(c) for c in binding)
+                variables.add(StateVariableLite(symbol, binding_with_constants))
+        return variables
+
+    def get_applicable_actions(self, s):
+        return [a for a in self.actions if entails(s, a.pres)]
+
+    def ground_problem(self, problem):
+        grounder = LPGroundingStrategy(problem, include_variable_inequalities=True)
+        action_groundings = grounder.ground_actions()
+        operators = []
+        for action_name, groundings in action_groundings.items():
+            action = problem.get_action(action_name)
+            for grounding in groundings:
+                operators.append(
+                    ground_schema_into_plain_operator_from_grounding(action, grounding)
+                )
+
+        grounded_fluents = set(
+            [
+                grounded_fluent.to_atom()
+                for grounded_fluent in grounder.ground_state_variables().objects
+            ]
+        )
+        init = [f for f in problem.init.as_atoms() if f in grounded_fluents]
+        if isinstance(problem.goal, tarski.syntax.Atom):
+            goal = [problem.goal]
+        else:
+            goal = [f for f in problem.goal.subformulas if f in grounded_fluents]
+
+        return (grounded_fluents, init, goal, operators, grounder)
+
+    def get_static(self):
+        static_symbols = self.static_symbols()
+        ret = []
+        for symbol in static_symbols:
+            ret.extend(self.get_grounded_atoms(symbol))
+        return set([fix_name(str(x)) for x in ret])
+
+    def PDDL_replace_init_pddl_parser(self, s):
+        d = DomainParser()(open(self.domain_file, "r").read().lower())
+        p = ProblemParser()(open(self.problem_file, "r").read().lower())
+
+        new_state = get_atoms_pddl(d, p, s | self.get_static())
+
+        new_p = Problem(
+            p.name, domain=d, objects=p.objects, init=new_state, goal=p.goal
+        )
+
+        return d, new_p
+
+
+def parse_ans(response: str, parser: ACPGrammarParser, task: str):
+    return [parser.parse(clean_answer(resp, task)) for resp in response]
+
+
+# def parse_ans(response : str, parser : ACPGrammarParser, task : str):
+#     ans = [parser.parse(clean_answer(resp, task), debug=True) for resp in response]
+#     if any(elem is None for elem in ans) or any(elem is None for elem in ans[0]):
+#         return None
+#     return ans
+
+
+def remove_garbage(s):
+    while True:
+        if s.endswith("."):
+            s = s[:-1]
+        elif s.endswith("\n"):
+            s = s[:-2]
+        else:
+            break
+    return s.rstrip()
+
+
+def compare_str(s1, s2):
+    return remove_garbage(s1).lower() == remove_garbage(s2).lower()
+
+
+def compare(l1, l2):
+    if not isinstance(l1, list):
+        return compare_str(l1, l2)
+    if not isinstance(l2, list):
+        return False
+    for i, v in enumerate(l1):
+        if not compare(v, l2[i]):
+            return False
+    return True
+
+
+def check_prog_response(resp):
+    if (
+        "Positive Effects".lower() in resp.lower()
+        and "Negative Effects".lower() in resp.lower()
+    ):
+        if "[" not in resp:
+            return True
+    return False
+
+
+def clean_answer(resp, task):
+    # Minor cleanup
+    if "progression_gen" in task:
+        # Check for Positive Effects and Negative Effects instead of separation
+        if check_prog_response(resp):
+            # replace **Positive Effects** with "["
+            # replace **Negative Effects** with "] ["
+            # append "]" to the end
+            resp2 = resp.lower()
+            resp2 = resp2.replace("*", "")
+            resp2 = resp2.replace("positive effects", "[")
+            resp2 = resp2.replace("negative effects", "] [")
+            resp2 = resp2 + "]"
+            return resp2
+    if "action_justification_gen" in task:
+        # Check for "simplified plan:"
+        if "simplified plan:" in resp.lower():
+            resp2 = resp.lower()
+            resp2 = resp2.replace("*", "")
+            resp2 = resp2.split("simplified plan:")[1]
+            return resp2
+    return resp
+
+
+def get_grammar_task(task):
+    # print(task)
+    if task == "reachable_atom_gen":
+        return "act"
+    elif task == "progression_gen":
+        return "progression_list"
+    elif task == "validation_gen":
+        return "index"
+    elif task == "reachable_action_gen":
+        return "act"
+    elif task == "action_justification_gen":
+        return "action_list"
+    elif task == "landmarks_gen":
+        return "act"
+    elif task == "goal_closer_gen":
+        return "action_name"
+    elif task == "applicable_actions_gen":
+        return "action_list"
+
+
+##############################################################################
+#  Evaluators
+
+
+def fix_action_name(a):
+    assert a.startswith("(") and a.endswith(")")
+    return "(" + " ".join([x.strip() for x in a[1:-1].split(" ") if len(x) > 0]) + ")"
+
+
+def str_remove_before_first_parentheses(s):
+    if s.startswith("("):
+        return s
+    try:
+        return s[s.index("(") :]
+    except Exception:
+        return ""
+
+
+def str_remove_after_last_parentheses(s):
+    if s.endswith(")"):
+        return s
+
+    i = s.rfind(")")
+
+    if i == -1:
+        return ""
+    return s[: i + 1]
+
+
+def cleanup_answer(ans):
+    if isinstance(ans, str):
+        ans = str_remove_before_first_parentheses(ans)
+        ans = str_remove_after_last_parentheses(ans)
+        ans = ans.lower()
+        ans = (
+            ans.replace(")\n(", ")######(")
+            .replace("),(", ")######(")
+            .replace(") (", ")######(")
+            .split("######")
+        )
+        return ans
+    if isinstance(ans, list):
+        res = []
+        for x in ans:
+            res.extend(cleanup_answer(x))
+        return res
+
+
+def set_equal(ans1, ans2):
+    return set(ans1) == set(ans2)
+
+
+class BaseEvaluator(ABC):
+    def __init__(self) -> None:
+        self.scores = []
+
+    @abstractmethod
+    def get_score(self, ans, doc):
+        pass
+
+    def add_scores(self, scores):
+        self.scores.extend(scores)
+
+    def get_avg_score(self):
+        avg_score = sum(self.scores) / len(self.scores)
+        return avg_score
+
+
+def get_evaluator(group):
+    if group == "applicable_actions_gen":
+        return ApplicabilityEvaluator()
+    elif group == "progression_gen":
+        return ProgressionEvaluator()
+    elif group == "validation_gen":
+        return ValidationEvaluator()
+    elif group == "reachable_atom_gen":
+        return ReachabilityEvaluator()
+    elif group == "goal_closer_gen":
+        return NextActionEvaluator()
+    elif group == "action_justification_gen":
+        return JustificationEvaluator()
+    elif group == "landmarks_gen":
+        return LandmarksEvaluator()
+    elif group == "reachable_action_gen":
+        return ActionReachabilityEvaluator()
+    assert True, f"Group {group} not found"
+
+
+"""
+Action Reachability task: generate a valid action that is not applicable to any reachable state.
+answer: A subset of actions that are known to be unreachable (not an exhaustive set).
+        It is empty only when we *know* that there are no such actions.
+"""
+
+
+class ActionReachabilityEvaluator(BaseEvaluator):
+    def get_score(self, ans, doc):
+        real_answer = doc["answer"]
+        if not real_answer or len(real_answer) == 0:
+            # The correct answer is None
+            self.add_scores(
+                ["none" == x.strip().lower() if x is not None else False for x in ans]
+            )
+        else:
+            for x in ans:
+                if x is None:
+                    self.scores.append(False)
+                    continue
+                action = x.strip().lower()
+                if action in real_answer:
+                    # The answer is in the subset of stored correct answers
+                    self.scores.append(True)
+                    continue
+                prec = get_action_preconditions(
+                    doc["PDDL_domain"].lower(), doc["PDDL_problem"].lower(), action
+                )
+                if prec is None:
+                    # The answer does not correspond to a valid action
+                    self.scores.append(False)
+                else:
+                    # Need to run a planner on a task with the answer action preconditions as the new goal
+                    prec = f"(and {' '.join(prec)})"
+                    self.scores.append(
+                        is_unsolvable_new_goal(
+                            doc["PDDL_domain"].lower(),
+                            doc["PDDL_problem"].lower(),
+                            prec,
+                        )
+                    )
+
+        return self.get_avg_score()
+
+
+"""
+Action Applicability task: generate all actions that are applicable in the current state.
+answer: A set of all applicable actions.
+"""
+
+
+class ApplicabilityEvaluator(BaseEvaluator):
+    def get_score(self, ans, doc):
+        real_answer = doc["answer"]
+        real_answer = [a.lower() for a in real_answer]
+        ans = [[fix_action_name(a) for a in x] if x is not None else None for x in ans]
+
+        # Check if the answer is equal (as a set) to the real stored answer
+        self.add_scores(
+            [
+                set_equal(real_answer, cleanup_answer(x)) if x is not None else False
+                for x in ans
+            ]
+        )
+        return self.get_avg_score()
+
+
+def is_subsequence(plan, new_plan):
+    i = 0
+    for a in plan:
+        if a == new_plan[i]:
+            i += 1
+            if len(new_plan) == i:
+                # Done
+                return True
+    return False
+
+
+def is_subsequence_and_plan(domain, problem, plan, new_plan):
+    if len(plan) <= len(new_plan):
+        return False
+    if not is_subsequence(plan, new_plan):
+        return False
+    return is_plan(domain, problem, new_plan)
+
+
+"""
+Justification task: generate a proper subsequence of the given plan that is also a plan.
+answer: A list of examples of actions that can be removed (ignored in evaluation).
+"""
+
+
+class JustificationEvaluator(BaseEvaluator):
+    def get_score(self, ans, doc):
+        # Sequence of actions (plan) from the question
+        if "inputs" in doc:  # old field name
+            seq = doc["inputs"][19:-147]
+        else:
+            seq = doc["question"][19:-147]
+        seq = seq.replace(") (", ")######(").split("######")
+        for x in ans:
+            if x is None:
+                self.scores.append(False)
+                continue
+            # An answer plan candidate
+            x = [fix_action_name(a) for a in x]
+            if len(x) == 0:
+                # Wrong answer - never an empty sequence
+                self.scores.append(0)
+                continue
+            # Check if the plan candidate from the answer (a) is a proper subsequence of the plan in the question and (b) is a plan.
+            self.scores.append(
+                is_subsequence_and_plan(
+                    doc["PDDL_domain"].lower(), doc["PDDL_problem"].lower(), seq, x
+                )
+            )
+        return self.get_avg_score()
+
+
+"""
+Landmarks task: generate a fact that is a non-trivial landmark for the current state.
+answer: A list of facts that are found to be landmarks and a list of facts that are found to be non-landmarks.
+
+The questions are generated only for cases where all facts either
+    (a) hold in the current state,
+    (b) true in goal,
+    (c) are found to be landmarks, or
+    (d) are found to be non-landmarks.
+In such cases, the evaluation is simple, it does not require checking whether a fact is a landmark, it was
+already done during question generation.
+"""
+
+
+class LandmarksEvaluator(BaseEvaluator):
+    def get_score(self, ans, doc):
+        # The set of facts that are found to be landmarks
+        real_answer = doc["answer"]
+        real_answer_yes = [a.lower() for a in real_answer["yes"]]
+
+        for x in ans:
+            if x is None:
+                self.scores.append(False)
+                continue
+            if x.strip().lower() in real_answer_yes:
+                # The answer fact is known to be landmark
+                self.scores.append(True)
+            elif x.strip().lower() == "none":
+                # The answer is none, correct only if there are no known landmarks,
+                #   since we only generate questions when that means that there are no non-trivial landmarks
+                self.scores.append(len(real_answer_yes) == 0)
+            else:
+                # All other cases the answer is incorrect
+                self.scores.append(False)
+
+        return self.get_avg_score()
+
+
+"""
+Next Action task: generate an action that takes us closer to the goal.
+answer:
+    (a) A list of applicable actions that are known to be correct answers
+    (b) A list of applicable actions that are known to be incorrect answers
+    (c) The rest of the applicable actions (maybe).
+"""
+
+
+class NextActionEvaluator(BaseEvaluator):
+    def get_score(self, ans, doc):
+        real_answer = doc["answer"]
+        real_answer_yes = [a.lower() for a in real_answer["yes"]]
+        real_answer_no = [a.lower() for a in real_answer["no"]]
+        real_answer_maybe = [a.lower() for a in real_answer["maybe"]]
+        # The cost of the optimal plan from the current state
+        opt = real_answer.get("opt", None)
+        for x in ans:
+            if x is None:
+                self.scores.append(False)
+                continue
+            action = x.strip().lower()
+            if action in real_answer_yes:
+                # Known to be correct
+                self.scores.append(True)
+            elif action in real_answer_no:
+                # Known to be incorrect
+                self.scores.append(False)
+            elif action not in real_answer_maybe:
+                # Not applicable, must be incorrect
+                self.scores.append(False)
+            else:
+                # Unknown, need to run a planner to check whether the state that results from applying the action is closer to the goal
+                #  meaning has smaller optimal plan cost.
+                self.scores.append(
+                    is_on_optimal_plan(
+                        doc["PDDL_domain"].lower(),
+                        doc["PDDL_problem"].lower(),
+                        action,
+                        opt,
+                    )
+                )
+
+        return self.get_avg_score()
+
+
+"""
+Progression task: generate the positive and negative effects of an action in the current state.
+answer:
+    (a) A list of facts that were false and become true, when the action is applied
+    (b) A list of facts that were true and become false, when the action is applied
+"""
+
+
+class ProgressionEvaluator(BaseEvaluator):
+    def get_score(self, ans, doc):
+        real_answer = doc["answer"]
+        real_answer_pos = [a.lower() for a in real_answer["pos"]]
+        real_answer_neg = [a.lower() for a in real_answer["neg"]]
+
+        for x in ans:
+            # The answer should be two lists. We allow for a single list and assume that the second one is empty (relaxed evaluation).
+            if x is None or len(x) > 2 or len(x) < 1:
+                self.scores.append(False)
+            else:
+                p = cleanup_answer(x[0])
+                if len(x) == 2:
+                    n = cleanup_answer(x[1])
+                else:
+                    # Assuming the last element is dropped because it is empty
+                    n = []
+                # Check if the answer is equal as sets to the correct answers.
+                ans = [set_equal(real_answer_pos, p), set_equal(real_answer_neg, n)]
+                self.scores.append(all(ans))
+
+        return self.get_avg_score()
+
+
+"""
+Reachability task: generate a valid fact that will never become true in any reachable state.
+answer: A subset of facts that are known to be unreachable (not an exhaustive set).
+        It is empty only when we *know* that there are no such facts.
+"""
+
+
+class ReachabilityEvaluator(BaseEvaluator):
+    def get_score(self, ans, doc):
+        real_answer = doc["answer"]
+        real_answer = [f"({x.strip().lower()})" for x in real_answer]
+
+        if len(real_answer) == 0:
+            # The correct answer is None
+            self.add_scores(
+                ["none" == x.strip().lower() if x is not None else False for x in ans]
+            )
+        else:
+            for x in ans:
+                if x is None:
+                    self.scores.append(False)
+                elif x.strip().lower() in real_answer:
+                    # The answer is in the subset of stored correct answers
+                    self.scores.append(True)
+                else:
+                    # Need to run a planner on a task with the answer fact as the new goal
+                    atom = x.strip().lower()
+                    self.scores.append(
+                        is_unsolvable_new_goal(
+                            doc["PDDL_domain"].lower(),
+                            doc["PDDL_problem"].lower(),
+                            atom,
+                        )
+                    )
+
+        return self.get_avg_score()
+
+
+"""
+Validation task: generate an index of the first inapplicable action in the given sequence.
+answer: the correct index.
+"""
+
+
+class ValidationEvaluator(BaseEvaluator):
+    def get_score(self, ans, doc):
+        real_answer = str(doc["answer"])
+        assert int(real_answer) >= 0, (
+            f"The index must be non-negative, received {real_answer}"
+        )
+        # Exact match
+        self.add_scores(
+            [
+                real_answer.lower() == x.strip().lower() if x is not None else False
+                for x in ans
+            ]
+        )
+
+        return self.get_avg_score()
+
+
+##############################################################################
+
+
+def dump_item(item, **kwargs):
+    return json.dumps(item)
+
+
+def parse_prediction(prediction):
+    try:
+        ans = json.loads(prediction.strip())
+        response = ans.get("answer", None)
+        return response
+    except Exception as e:
+        print(f"Exception occurred {e}")
+        return prediction
+
+
+@register_filter("ACP_grammar_filter")
+class ACPGrammarFilter(RegexFilter):
+    """Filtering Index using"""
+
+    def __init__(self, *args, **kwargs):
+        self.parser = ACPGrammarParser(kwargs["grammar_task"])
+        self.clean = kwargs["clean"] if "clean" in kwargs else None
+
+    def clean_pos_neg(self, resp):
+        # Check for Positive Effects and Negative Effects instead of separation
+        if check_prog_response(resp):
+            resp2 = resp.lower()
+            resp2 = resp2.replace("*", "")
+            resp2 = resp2.replace("positive effects", "[")
+            resp2 = resp2.replace("negative effects", "] [")
+            resp2 = resp2 + "]"
+            return resp2
+        return resp
+
+    def clean_simplified_plan(self, resp):
+        # Check for "simplified plan:"
+        if "simplified plan:" in resp.lower():
+            resp2 = resp.lower()
+            resp2 = resp2.replace("*", "")
+            resp2 = resp2.split("simplified plan:")[1]
+            return resp2
+        return resp
+
+    def apply(self, resps, docs):
+        if self.clean == "pos_neg":
+            filtered_resps = [
+                [self.parser.parse(self.clean_pos_neg(r)) for r in resp]
+                for resp in resps
+            ]
+        elif self.clean == "simplified plan":
+            filtered_resps = [
+                [self.parser.parse(self.clean_simplified_plan(r)) for r in resp]
+                for resp in resps
+            ]
+        else:
+            filtered_resps = [[self.parser.parse(r) for r in resp] for resp in resps]
+        return filtered_resps
+
+
+def process_acp_results(doc, results):
+    return {"score": get_evaluator(doc["group"]).get_score(results, doc)}
+
+
+def get_score(references, predictions, **kwargs):
+    # print(f"References: {references}")
+    # print(f"Predictions: {predictions}")
+    data = json.loads(references[0].strip())
+    real_ans = data["answer"]
+    task = data["group"]
+
+    responses = [parse_prediction(prediction) for prediction in predictions]
+
+    print(f"Real answer: {real_ans}")
+    print(f"Model answers: {responses}")
+    parser = ACPGrammarParser(get_grammar_task(task))
+    ans = parse_ans(responses, parser, task)
+
+    print(f"Parsed model answers: {ans}")
+    score = get_evaluator(task).get_score(ans, data)
+
+    return {"get_score": score}

lm-evaluation-harness/lm_eval/tasks/acpbench/gen_2shot/act_reach.yaml

@@ -0,0 +1,19 @@
+task: acp_areach_gen
+dataset_name: acp_areach_gen
+include: _gen_yaml_2shot
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty.  There are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0.  Currently, the robot is at position f2-2f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-0 is at position f1-2f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with key ?key of shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - move from place ?curpos to place ?nextpos, (pickup ?curpos ?key) - retrieve the key ?key from its current position ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey from the current position ?curpos and loose the key ?oldkey which is being held, and (putdown ?curpos ?key) - put the key ?key at the current position place ?curpos."
+    question: "What action can never become applicable, in any state reachable from the current state?"
+    answer: "(unlock f0-3f f0-4f key0-0 shape0)"
+  - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-1 and l1-0 are in c1; l0-0 and l0-1 are in c0. Currently, t1 is at l1-1, a0 is at l1-0, p0 is at l0-0, t0 is at l0-1, p2 is in a0, p1 is in t1, p3 is in t0. The available actions are: (load-truck ?obj ?truck ?loc) - load the object ?obj from location ?loc into the truck ?truck, (load-airplane ?obj ?airplane ?loc) - load object ?obj into airplane ?airplane at location ?loc, (unload-truck ?obj ?truck ?loc) - offload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - offload the object ?obj from the airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from location ?loc-from in city ?city to location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - operate the airplane ?airplane from airport ?loc-from to airport ?loc-to."
+    question: "What action can never become applicable, in any state reachable from the current state?"
+    answer: "(drive-truck t0 l1-1 l0-0 c0)"
+doc_to_text: "**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide one action or None. **Final Answer**:"
+filter_list:
+  - name: "acp_grammar_parse"
+    filter:
+      - function: "ACP_grammar_filter"
+        grammar_task: "act"
+      - function: "take_first"

lm-evaluation-harness/lm_eval/tasks/acpbench/gen_2shot/app.yaml

@@ -0,0 +1,19 @@
+task: acp_app_gen
+dataset_name: acp_app_gen
+include: _gen_yaml_2shot
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty.  There are 2 keys in 1 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0.  Currently, the robot is at position f3-2f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-0 is at position f2-2f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with key ?key of shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - travel from the current position ?curpos to the next position ?nextpos, (pickup ?curpos ?key) - pick up key ?key from place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey at the current position place ?curpos and loose the key ?oldkey being held, and (putdown ?curpos ?key) - put down key ?key at current position place ?curpos."
+    question: "Generate the list of all ground actions that are applicable in this state."
+    answer: "[(move f3-2f f3-1f), (move f3-2f f2-2f), (move f3-2f f3-3f)]"
+  - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l0-0 and l0-1 are in c0; l1-0 and l1-1 are in c1. Currently, t1, p2, and p3 are at l1-0, a0 is at l0-0, t0 is at l0-1, p1 and p0 are in t1. The available actions are: (load-truck ?obj ?truck ?loc) - load object ?obj into truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load object ?obj into airplane ?airplane at location ?loc, (unload-truck ?obj ?truck ?loc) - unload object ?obj from truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - remove the object ?obj from the airplane ?airplane and place it on the location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from its current location ?loc-from in city ?city to the new location ?loc-to within the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly airplane ?airplane from airport ?loc-from to airport ?loc-to."
+    question: "Generate the list of all ground actions that are applicable in this state."
+    answer: "[(drive-truck t1 l1-0 l1-0 c1), (drive-truck t0 l0-1 l0-0 c0), (load-truck p2 t1 l1-0), (unload-truck p0 t1 l1-0), (drive-truck t0 l0-1 l0-1 c0), (fly-airplane a0 l0-0 l1-0), (fly-airplane a0 l0-0 l0-0), (unload-truck p1 t1 l1-0), (drive-truck t1 l1-0 l1-1 c1), (load-truck p3 t1 l1-0)]"
+doc_to_text: "**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide only the actions. **Final Answer**:"
+filter_list:
+  - name: "acp_grammar_parse"
+    filter:
+      - function: "ACP_grammar_filter"
+        grammar_task: "action_list"
+      - function: "take_first"

lm-evaluation-harness/lm_eval/tasks/acpbench/gen_2shot/just.yaml

@@ -0,0 +1,20 @@
+task: acp_just_gen
+dataset_name: acp_just_gen
+include: _gen_yaml_2shot
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty.  There are 2 keys in 1 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0.  Currently, the robot is at position f3-3f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock, f2-0f has shape0 shaped lock. Key key0-0 is at position f2-2f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock place ?lockpos with key ?key of shape ?shape from current position place ?curpos, (move ?curpos ?nextpos) - move from ?curpos to ?nextpos, (pickup ?curpos ?key) - retrieve the key ?key from its current position ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up key ?newkey at current position place ?curpos and loose key ?oldkey being held, and (putdown ?curpos ?key) - put down the key ?key at the current position ?curpos. The goal is to reach a state where the following facts hold: Key key0-0 is at f2-0f location and Key key0-1 is at f1-3f location."
+    question: "Simplify the plan [(move f3-3f f3-2f), (move f3-2f f2-2f), (pickup f2-2f key0-0), (move f2-2f f2-1f), (putdown f2-1f key0-0), (pickup f2-1f key0-0), (unlock f2-1f f2-0f key0-0 shape0), (move f2-1f f2-0f), (putdown f2-0f key0-0)] by removing either a single action or a pair of consecutive actions, while still maintaining a valid plan. Provide the resulting simplified plan."
+    answer: "[(move f3-3f f3-2f), (move f3-2f f2-2f), (pickup f2-2f key0-0), (move f2-2f f2-1f), (unlock f2-1f f2-0f key0-0 shape0), (move f2-1f f2-0f), (putdown f2-0f key0-0)]"
+  - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-1 and l1-0 are in c1; l0-0 and l0-1 are in c0. Currently, p2, p1, and p3 are at l1-0, p0 and t1 are at l1-1, t0 is at l0-1, a0 is at l0-0. The available actions are: (load-truck ?obj ?truck ?loc) - load the object ?obj from location ?loc into the truck ?truck, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc onto the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - remove the object ?obj from the airplane ?airplane and place it on the location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - drive truck ?truck from location ?loc-from in city ?city to location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly the airplane ?airplane from location ?loc-from to location ?loc-to. The goal is to reach a state where the following facts hold: p3 is at l0-1, p2 is at l1-0, p0 is at l0-0, and p1 is at l1-0."
+    question: "Simplify the plan [(fly-airplane a0 l0-0 l1-0), (fly-airplane a0 l1-0 l0-0), (load-truck p0 t1 l1-1), (drive-truck t1 l1-1 l1-0 c1), (unload-truck p0 t1 l1-0), (fly-airplane a0 l0-0 l1-0), (load-airplane p0 a0 l1-0), (load-airplane p3 a0 l1-0), (fly-airplane a0 l1-0 l0-0), (unload-airplane p0 a0 l0-0), (unload-airplane p3 a0 l0-0), (drive-truck t0 l0-1 l0-0 c0), (load-truck p3 t0 l0-0), (drive-truck t0 l0-0 l0-1 c0), (unload-truck p3 t0 l0-1)] by removing either a single action or a pair of consecutive actions, while still maintaining a valid plan. Provide the resulting simplified plan."
+    answer: "[(load-truck p0 t1 l1-1), (drive-truck t1 l1-1 l1-0 c1), (unload-truck p0 t1 l1-0), (fly-airplane a0 l0-0 l1-0), (load-airplane p0 a0 l1-0), (load-airplane p3 a0 l1-0), (fly-airplane a0 l1-0 l0-0), (unload-airplane p0 a0 l0-0), (unload-airplane p3 a0 l0-0), (drive-truck t0 l0-1 l0-0 c0), (load-truck p3 t0 l0-0), (drive-truck t0 l0-0 l0-1 c0), (unload-truck p3 t0 l0-1)]"
+doc_to_text: "**Question**: {{context}} {{question}} **Final Answer**:"
+filter_list:
+  - name: "acp_grammar_parse"
+    filter:
+      - function: "ACP_grammar_filter"
+        grammar_task: "action_list"
+        clean: "simplified plan"
+      - function: "take_first"

lm-evaluation-harness/lm_eval/tasks/acpbench/gen_2shot/land.yaml

@@ -0,0 +1,19 @@
+task: acp_land_gen
+dataset_name: acp_land_gen
+include: _gen_yaml_2shot
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty.  There are 2 keys in 1 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0.  Currently, the robot is at position f3-0f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-0 is at position f3-0f. Key key0-1 is at position f1-3f. The goal is to reach a state where the following facts hold: Key key0-0 is at f2-0f location and Key key0-1 is at f1-3f location. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot's arm is empty."
+    question: "Generate a non-trivial fact landmark, one that does not hold in the initial state or goal."
+    answer: "(holding key0-0)"
+  - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-0 and l1-1 are in c1; l0-1 and l0-0 are in c0. Currently, a0 and p2 are at l1-0, t0 is at l0-0, t1 is at l1-1, p3 and p1 are in a0, p0 is in t1. The goal is to reach a state where the following facts hold: p0 is at l0-0, p2 is at l1-0, p1 is at l1-0, and p3 is at l0-1. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2."
+    question: "Generate a non-trivial fact landmark, one that does not hold in the initial state or goal."
+    answer: "(in p3 t0)"
+doc_to_text: "**Question**: {{context}} {{question}} Provide only the ground proposition or None. **Final Answer**:"
+filter_list:
+  - name: "acp_grammar_parse"
+    filter:
+      - function: "ACP_grammar_filter"
+        grammar_task: "act"
+      - function: "take_first"

lm-evaluation-harness/lm_eval/tasks/acpbench/gen_2shot_with_pddl/_gen_yaml_2shot

@@ -0,0 +1,26 @@
+tag:
+  - acp_gen_2shot_with_pddl
+  - acp_bench_hard_with_pddl
+dataset_path: ibm-research/acp_bench
+test_split: test
+description: "Answer the question based on the provided PDDL domain and PDDL problem. The current state is the initial state described in the PDDL problem below.\n\n"
+doc_to_target: "{{answer}}"
+output_type: generate_until
+num_fewshot: 2
+generation_kwargs:
+  until:
+    - "\n\n\n\n"
+    - "\n\n"
+    - "**Question**:"
+    - "**Question:**"
+    - "Q:"
+  do_sample: false
+  max_gen_toks: 1000
+  temperature: 0.0
+metadata:
+  version: 1.0
+process_results: !function acp_utils.process_acp_results
+metric_list:
+  - metric: "score"
+    aggregation: mean
+    higher_is_better: True

lm-evaluation-harness/lm_eval/tasks/acpbench/gen_2shot_with_pddl/act_reach.yaml

@@ -0,0 +1,23 @@
+task: acp_areach_gen_with_pddl
+dataset_name: acp_areach_gen
+include: _gen_yaml_2shot
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty.  \nThere are 2 keys in 0 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0.  \nCurrently, the robot is at position f3-2f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f2-2f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with key ?key of shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - move to place ?nextpos from place ?curpos, (pickup ?curpos ?key) - acquire the key ?key from the place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up key ?newkey at current position place ?curpos and loose key ?oldkey being held, and (putdown ?curpos ?key) - put down key ?key at current position place ?curpos."
+    question: "What action can never become applicable, in any state reachable from the current state?"
+    answer: "(pickup-and-loose f0-1f key0-0 key0-0)"
+    PDDL_domain: "(define (domain grid)\n    (:requirements :strips :typing)\n    (:types key place shape - object)\n    (:predicates (arm-empty) (at ?r - key ?x - place)  (at-robot ?x - place)  (conn ?x - place ?y - place)  (holding ?k - key)  (key-shape ?k - key ?s - shape)  (lock-shape ?x - place ?s - shape)  (locked ?x - place)  (open ?x - place))\n    (:action move\n        :parameters (?curpos - place ?nextpos - place)\n        :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n        :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n    )\n     (:action pickup\n        :parameters (?curpos - place ?key - key)\n        :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n        :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n    )\n     (:action pickup-and-loose\n        :parameters (?curpos - place ?newkey - key ?oldkey - key)\n        :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n        :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n    )\n     (:action putdown\n        :parameters (?curpos - place ?key - key)\n        :precondition (and (at-robot ?curpos) (holding ?key))\n        :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n    )\n     (:action unlock\n        :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n        :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n        :effect (and (open ?lockpos) (not (locked ?lockpos)))\n    )\n)"
+    PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n    (:domain grid)\n    (:requirements :strips :typing)\n    (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n    (:init (arm-empty) (at key0-0 f2-2f) (at key0-1 f1-3f) (at-robot f3-2f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f2-0f) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n    (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)"
+  - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l1-0 and l1-1 are in c1; l0-0 and l0-1 are in c0. \nCurrently, a0, p1, and p2 are at l1-0, t0 is at l0-1, p3 and p0 are at l0-0, t1 is at l1-1. The available actions are: (load-truck ?obj ?truck ?loc) - place the object ?obj into the truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc into the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload object ?obj from truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - unload object ?obj from airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from its current location ?loc-from in city ?city to the new location ?loc-to within the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly airplane ?airplane from airport ?loc-from to airport ?loc-to."
+    question: "What action can never become applicable, in any state reachable from the current state??"
+    answer: "(load-truck p2 t0 l1-1)"
+    PDDL_domain: "(define (domain logistics-strips)\n  (:requirements :strips :typing) \n\n  (:types \n    location locatable city - object \n    package movable - locatable\n    airport - location\n    airplane truck - movable    \n  )\t\t\n  \n  (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n  :parameters\n   (?obj - package\n    ?truck - truck\n    ?loc - location)\n  :precondition\n   (and \n   (at ?truck ?loc) (at ?obj ?loc))\n  :effect\n   (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n  :parameters\n   (?obj - package\n    ?airplane - airplane\n    ?loc - location)\n  :precondition\n   (and \n   (at ?obj ?loc) (at ?airplane ?loc))\n  :effect\n   (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n  :parameters\n   (?obj - package\n    ?truck - truck\n    ?loc - location)\n  :precondition\n   (and \n        (at ?truck ?loc) (in ?obj ?truck))\n  :effect\n   (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n  :parameters\n   (?obj - package\n    ?airplane - airplane\n    ?loc - location)\n  :precondition\n   (and \n        (in ?obj ?airplane) (at ?airplane ?loc))\n  :effect\n   (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n  :parameters\n   (?truck - truck\n    ?loc-from - location\n    ?loc-to - location\n    ?city - city)\n  :precondition\n   (and \n   (at ?truck ?loc-from)\n   (in-city ?loc-from ?city)\n   (in-city ?loc-to ?city))\n  :effect\n   (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n  :parameters\n   (?airplane - airplane\n    ?loc-from - airport\n    ?loc-to - airport)\n  :precondition\n   (and \n\t(at ?airplane ?loc-from))\n  :effect\n   (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)"
+    PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n    (:domain logistics-strips)\n    (:requirements :strips :typing)\n    (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n    (:init (at a0 l1-0) (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-0) (at t0 l0-1) (at t1 l1-1) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n    (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)"
+doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide one action or None. **Final Answer**:"
+filter_list:
+  - name: "acp_grammar_parse"
+    filter:
+      - function: "ACP_grammar_filter"
+        grammar_task: "act"
+      - function: "take_first"

lm-evaluation-harness/lm_eval/tasks/acpbench/gen_2shot_with_pddl/land.yaml

@@ -0,0 +1,23 @@
+task: acp_land_gen_with_pddl
+dataset_name: acp_land_gen
+include: _gen_yaml_2shot
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty.  \nThere are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0.  \nCurrently, the robot is at position f1-0f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f1-0f. The goal is to reach a state where the following facts hold: Key key0-1 is at f1-3f location and Key key0-0 is at f2-0f location. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot's arm is empty."
+    question: "Generate a non-trivial fact landmark, one that does not hold in the initial state or goal."
+    answer: "(at-robot f2-0f)"
+    PDDL_domain: "(define (domain grid)\n    (:requirements :strips :typing)\n    (:types key place shape - object)\n    (:predicates (arm-empty) (at ?r - key ?x - place)  (at-robot ?x - place)  (conn ?x - place ?y - place)  (holding ?k - key)  (key-shape ?k - key ?s - shape)  (lock-shape ?x - place ?s - shape)  (locked ?x - place)  (open ?x - place))\n    (:action move\n        :parameters (?curpos - place ?nextpos - place)\n        :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n        :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n    )\n     (:action pickup\n        :parameters (?curpos - place ?key - key)\n        :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n        :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n    )\n     (:action pickup-and-loose\n        :parameters (?curpos - place ?newkey - key ?oldkey - key)\n        :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n        :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n    )\n     (:action putdown\n        :parameters (?curpos - place ?key - key)\n        :precondition (and (at-robot ?curpos) (holding ?key))\n        :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n    )\n     (:action unlock\n        :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n        :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n        :effect (and (open ?lockpos) (not (locked ?lockpos)))\n    )\n)"
+    PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n    (:domain grid)\n    (:requirements :strips :typing)\n    (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n    (:init (arm-empty) (at key0-0 f1-0f) (at key0-1 f1-3f) (at-robot f1-0f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-0f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n    (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)"
+  - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l0-1 and l0-0 are in c0; l1-0 and l1-1 are in c1. \nCurrently, a0 and t0 are at l0-0, t1 is at l1-1, p2, p1, and p3 are at l1-0, p0 is in t1. The goal is to reach a state where the following facts hold: p1 is at l1-0, p0 is at l0-0, p2 is at l1-0, and p3 is at l0-1. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2."
+    question: "Generate a non-trivial fact landmark, one that does not hold in the initial state or goal."
+    answer: "(in p3 t0)"
+    PDDL_domain: "(define (domain logistics-strips)\n  (:requirements :strips :typing) \n\n  (:types \n    location locatable city - object \n    package movable - locatable\n    airport - location\n    airplane truck - movable    \n  )\t\t\n  \n  (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n  :parameters\n   (?obj - package\n    ?truck - truck\n    ?loc - location)\n  :precondition\n   (and \n   (at ?truck ?loc) (at ?obj ?loc))\n  :effect\n   (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n  :parameters\n   (?obj - package\n    ?airplane - airplane\n    ?loc - location)\n  :precondition\n   (and \n   (at ?obj ?loc) (at ?airplane ?loc))\n  :effect\n   (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n  :parameters\n   (?obj - package\n    ?truck - truck\n    ?loc - location)\n  :precondition\n   (and \n        (at ?truck ?loc) (in ?obj ?truck))\n  :effect\n   (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n  :parameters\n   (?obj - package\n    ?airplane - airplane\n    ?loc - location)\n  :precondition\n   (and \n        (in ?obj ?airplane) (at ?airplane ?loc))\n  :effect\n   (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n  :parameters\n   (?truck - truck\n    ?loc-from - location\n    ?loc-to - location\n    ?city - city)\n  :precondition\n   (and \n   (at ?truck ?loc-from)\n   (in-city ?loc-from ?city)\n   (in-city ?loc-to ?city))\n  :effect\n   (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n  :parameters\n   (?airplane - airplane\n    ?loc-from - airport\n    ?loc-to - airport)\n  :precondition\n   (and \n\t(at ?airplane ?loc-from))\n  :effect\n   (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)"
+    PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n    (:domain logistics-strips)\n    (:requirements :strips :typing)\n    (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n    (:init (at a0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l1-0) (at t0 l0-0) (at t1 l1-1) (in p0 t1) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n    (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)"
+doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Provide only the ground proposition or None. **Final Answer**:"
+filter_list:
+  - name: "acp_grammar_parse"
+    filter:
+      - function: "ACP_grammar_filter"
+        grammar_task: "act"
+      - function: "take_first"

lm-evaluation-harness/lm_eval/tasks/acpbench/mcq_cot_2shot/just.yaml

@@ -0,0 +1,12 @@
+task: acp_just_mcq
+dataset_name: acp_just_mcq
+include: _mcq_cot_2shot_yaml
+fewshot_config:
+  sampler: first_n
+  samples:
+  - context: 'This is a ferry domain, where the task is to transport cars from their start to their goal locations, using a ferry. Each location is accessible by ferry from each other location. The cars can be debarked or boarded, and the ferry can carry only one car at a time.  There are 2 locations and 2 cars, numbered consecutively.  Currently, the ferry is at l0 location and it is empty. The cars are at locations as follows: c0 and c1 are at l0. The goal is to reach a state where the following facts hold: Car c1 is at location l1 and Car c0 is at location l1.'
+    question: 'Given the plan: \"board the car c0 at the location l0, travel by sea from location l0 to location l1, unload the car c0 from the ferry to location l1, travel by sea from location l1 to location l0, board the car c1 at location l0, sail from location l0 to location l1, debark the car c1 from the ferry to location l1, travel by sea from location l1 to location l0\"; which of the following actions can be removed from this plan and still have a valid plan?  A. board the car c1 at the location l0.  B. debark car c1 to location l1 from the ferry.  C. travel by sea from location l1 to location l0.  D. sail from location l0 to location l1.'
+    answer: "Let's think step by step.   Step 1: A valid plan is a sequence of actions that is exectuable from the current state and achieves the goal.  Step 2: The 2nd instance of  \"travel by sea from location l1 to location l0\" can be removed from the plan and still have a valid plan. **Final Answer**: C."
+  - context: 'There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations.  There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities.  The locations are in cities as follows: l0-0 and l0-1 are in c0; l1-0 and l1-1 are in c1.  Currently, p2, p1, and p3 are at l1-0, p0 and t1 are at l1-1, a0 is at l0-0, t0 is at l0-1. The goal is to reach a state where the following facts hold: p3 is at l0-1, p0 is at l0-0, p2 is at l1-0, and p1 is at l1-0.'
+    question: 'Given the plan: \"load object p0 into truck t1 at location l1-1, navigate the truck t1 which is in location l1-1 in city c1 to another location l1-0 in the same city, offload the object p0 from the truck t1 at location l1-0, fly the airplane a0 from location l0-0 to location l1-0, load object p0 into airplane a0 at location l1-0, place the object p3 onto the airplane a0 at location l1-0, fly the airplane a0 from location l1-0 to location l0-0, unload the object p0 from the airplane a0 at location l0-0, unload the object p3 from the airplane a0 at location l0-0, navigate the truck t0 from location l0-1 in city c0 to location l0-0 in the same city, load object p3 into truck t0 at location l0-0, drive truck t0 from location l0-0 in city c0 to location l0-1 in the same city, offload the object p3 from the truck t0 at location l0-1, drive truck t0 from location l0-1 in city c0 to location l0-0 in the same city, navigate the truck t0 from its current location l0-0 in city c0 to the new location l0-1 within the same city\"; which of the following pair of consecutive actions can be removed from this plan and still have a valid plan?  A. drive the truck t0 in city c0 from location l0-1 to location l0-0 and navigate the truck t0 which is in location l0-0 in city c0 to another location l0-1 in the same city.  B. unload object p3 from truck t0 at location l0-1 and drive truck t0 from location l0-1 in city c0 to location l0-0 in the same city.  C. drive truck t0 from location l0-1 in city c0 to location l0-0 in the same city and load the object p3 from location l0-0 into the truck t0.  D. unload the object p0 from the airplane a0 at location l0-0 and remove the object p3 from the airplane a0 and place it on the location l0-0.'
+    answer: "Let's think step by step.   Step 1: A valid plan is a sequence of actions that is exectuable from the current state and achieves the goal.  Step 2: Consecutive actions \"drive the truck t0 in city c0 from location l0-1 to location l0-0 and navigate the truck t0 which is in location l0-0 in city c0 to another location l0-1 in the same city\" can be removed from the plan and still have a valid plan. **Final Answer**: A."

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yaml

@@ -0,0 +1,33 @@
+tag:
+    - afrimgsm_tasks
+    - afrimgsm_tasks_prompt_4
+dataset_path: masakhane/afrimgsm
+output_type: generate_until
+test_split: test
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
+target_delimiter: ""
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+  - filter:
+    - function: regex
+      group_select: -1
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
+    - function: take_first
+    name: flexible-extract
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 2.0

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_zul.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\
+  \ that the response is clear and without any supplementary information. \n\nQuestion:\
+  \ {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_zul_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sna.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: sna
+doc_to_text: "For mathematical questions provided in chiShona language. Supply the\
+  \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\
+  Answer: "
+include: afrimgsm_yaml
+task: afrimgsm_sna_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sot.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: sot
+doc_to_text: "For mathematical questions provided in Sesotho language. Supply the\
+  \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\
+  Answer: "
+include: afrimgsm_yaml
+task: afrimgsm_sot_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_twi.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: twi
+doc_to_text: "For mathematical questions provided in Twi language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_twi_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_vai.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: vai
+doc_to_text: "For mathematical questions provided in Vai language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_vai_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yaml

@@ -0,0 +1,33 @@
+tag:
+    - afrimgsm_tasks
+    - afrimgsm_tasks_prompt_5
+dataset_path: masakhane/afrimgsm
+output_type: generate_until
+test_split: test
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
+target_delimiter: ""
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+  - filter:
+    - function: regex
+      group_select: -1
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
+    - function: take_first
+    name: flexible-extract
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 2.0

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yor.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: "For mathematical questions provided in Yoruba language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_yor_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_zul.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: "For mathematical questions provided in Zulu language. Supply the accurate\
+  \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: "
+include: afrimgsm_yaml
+task: afrimgsm_zul_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lin.yaml

@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lin
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_lin_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lug.yaml

@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: lug
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_lug_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_swa.yaml

@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: swa
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_swa_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_twi.yaml

@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: twi
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_twi_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_vai.yaml

@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: vai
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_vai_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_wol.yaml

@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: wol
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_wol_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_xho.yaml

@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: xho
+include: afrimgsm_cot_yaml
+task: afrimgsm_cot_xho_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yaml

@@ -0,0 +1,37 @@
+tag:
+    - afrimgsm_cot_tasks
+    - afrimgsm_cot_tasks_prompt_1
+dataset_path: masakhane/afrimgsm
+dataset_name: null  # Overridden by language-specific config.
+output_type: generate_until
+training_split: train
+test_split: test
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+  - <|eot_id|>
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+filter_list:
+  - name: "strict-match"
+    filter:
+      - function: "regex"
+        regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
+      - function: "take_first"
+  - filter:
+    - function: regex
+      group_select: -1
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
+    - function: take_first
+    name: flexible-extract
+metadata:
+  version: 2.0