Spaces:
Sleeping
Sleeping
File size: 2,944 Bytes
1a212f3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | import re
from collections.abc import Callable
SPAN_PATTERN = re.compile(r"\{\{(.*?)\}\}")
TokenizeFn = Callable[[str], tuple[str, ...]]
def is_pass_response(content: str) -> bool:
stripped = content.strip()
candidate = stripped
if len(candidate) >= 2 and candidate[0] == candidate[-1] and candidate[0] in "\"'":
candidate = candidate[1:-1].strip()
return candidate.casefold() == "pass"
def strip_markers(marked_text: str) -> tuple[str, list[tuple[int, int]]]:
"""
Remove {{ }} markers and return cleaned text plus dissimilar character ranges
in the cleaned text.
"""
cleaned_parts: list[str] = []
dissimilar_ranges: list[tuple[int, int]] = []
cleaned_offset = 0
last_end = 0
for match in SPAN_PATTERN.finditer(marked_text):
before = marked_text[last_end:match.start()]
cleaned_parts.append(before)
cleaned_offset += len(before)
span_text = match.group(1)
span_start = cleaned_offset
span_end = span_start + len(span_text)
dissimilar_ranges.append((span_start, span_end))
cleaned_parts.append(span_text)
cleaned_offset = span_end
last_end = match.end()
cleaned_parts.append(marked_text[last_end:])
return "".join(cleaned_parts), dissimilar_ranges
def token_character_spans(text: str, tokens: tuple[str, ...]) -> list[tuple[int, int]] | None:
"""Locate each token in order within text. Returns None if a token cannot be found."""
spans: list[tuple[int, int]] = []
position = 0
for token in tokens:
index = text.find(token, position)
if index < 0:
return None
spans.append((index, index + len(token)))
position = index + len(token)
return spans
def extract_labels_from_marked(
original_text: str,
marked_text: str,
tokenize: TokenizeFn,
) -> tuple[int, ...]:
"""
Convert a {{marked}} copy of original_text into per-token dissimilar labels.
Requires only that cleaned and original token counts match (token strings may
differ, e.g. apostrophe variants). Length mismatch yields all zeros.
"""
original_tokens = tokenize(original_text)
if is_pass_response(marked_text):
return tuple(0 for _ in original_tokens)
cleaned_text, dissimilar_ranges = strip_markers(marked_text)
cleaned_tokens = tokenize(cleaned_text)
if len(cleaned_tokens) != len(original_tokens):
return tuple(0 for _ in original_tokens)
character_spans = token_character_spans(cleaned_text, cleaned_tokens)
if character_spans is None:
return tuple(0 for _ in original_tokens)
labels: list[int] = []
for start, end in character_spans:
is_dissimilar = any(
start < span_end and end > span_start
for span_start, span_end in dissimilar_ranges
)
labels.append(1 if is_dissimilar else 0)
return tuple(labels)
|