| import string |
| from typing import Any, List |
|
|
| from langchain.evaluation.schema import StringEvaluator |
|
|
|
|
| class ExactMatchStringEvaluator(StringEvaluator): |
| """Compute an exact match between the prediction and the reference. |
| |
| Examples |
| ---------- |
| >>> evaluator = ExactMatchChain() |
| >>> evaluator.evaluate_strings( |
| prediction="Mindy is the CTO", |
| reference="Mindy is the CTO", |
| ) # This will return {'score': 1.0} |
| |
| >>> evaluator.evaluate_strings( |
| prediction="Mindy is the CTO", |
| reference="Mindy is the CEO", |
| ) # This will return {'score': 0.0} |
| """ |
|
|
| def __init__( |
| self, |
| *, |
| ignore_case: bool = False, |
| ignore_punctuation: bool = False, |
| ignore_numbers: bool = False, |
| **kwargs: Any, |
| ): |
| super().__init__() |
| self.ignore_case = ignore_case |
| self.ignore_punctuation = ignore_punctuation |
| self.ignore_numbers = ignore_numbers |
|
|
| @property |
| def requires_input(self) -> bool: |
| """ |
| This evaluator does not require input. |
| """ |
| return False |
|
|
| @property |
| def requires_reference(self) -> bool: |
| """ |
| This evaluator requires a reference. |
| """ |
| return True |
|
|
| @property |
| def input_keys(self) -> List[str]: |
| """ |
| Get the input keys. |
| |
| Returns: |
| List[str]: The input keys. |
| """ |
| return ["reference", "prediction"] |
|
|
| @property |
| def evaluation_name(self) -> str: |
| """ |
| Get the evaluation name. |
| |
| Returns: |
| str: The evaluation name. |
| """ |
| return "exact_match" |
|
|
| def _evaluate_strings( |
| self, |
| *, |
| prediction: str, |
| reference: str, |
| **kwargs: Any, |
| ) -> dict: |
| """ |
| Evaluate the exact match between the prediction and the reference. |
| |
| Args: |
| prediction (str): The prediction string. |
| reference (Optional[str], optional): The reference string. |
| |
| Returns: |
| dict: The evaluation results containing the score. |
| """ |
| if self.ignore_case: |
| prediction = prediction.lower() |
| reference = reference.lower() |
| if self.ignore_punctuation: |
| prediction = prediction.translate(str.maketrans("", "", string.punctuation)) |
| reference = reference.translate(str.maketrans("", "", string.punctuation)) |
| if self.ignore_numbers: |
| prediction = prediction.translate(str.maketrans("", "", string.digits)) |
| reference = reference.translate(str.maketrans("", "", string.digits)) |
| return {"score": int(prediction == reference)} |
|
|
