File size: 942 Bytes
ce235ae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | """Check the pinned NanoJev renderer and tokenizer before a large export."""
from transformers import AutoTokenizer
from assets import snapshot
from fixtures import fixture
from preprocessing import prepare_request
def test_candidate_set_and_eos_positions():
root = snapshot(with_weights=False)
tokenizer = AutoTokenizer.from_pretrained(root / "tokenizer", local_files_only=True, trust_remote_code=False)
if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token
inputs, mask, example = prepare_request(root, tokenizer, fixture(), 128, 4)
assert example["candidate_ids"] == ["left", "right", "shoot", "noop"]
assert mask.tolist() == [[1, 1, 1, 1]]
assert inputs["attention_mask"].sum(axis=1).min() > 0
for row in range(4):
eos = inputs["eos_map"][row, 0].nonzero()[0]
assert len(eos) == 1
assert inputs["input_ids"][row, eos[0]] == tokenizer.eos_token_id
|