"""Check the pinned NanoJev renderer and tokenizer before a large export."""

from transformers import AutoTokenizer

from assets import snapshot
from fixtures import fixture
from preprocessing import prepare_request


def test_candidate_set_and_eos_positions():
    root = snapshot(with_weights=False)
    tokenizer = AutoTokenizer.from_pretrained(root / "tokenizer", local_files_only=True, trust_remote_code=False)
    if tokenizer.pad_token_id is None:
        tokenizer.pad_token = tokenizer.eos_token
    inputs, mask, example = prepare_request(root, tokenizer, fixture(), 128, 4)
    assert example["candidate_ids"] == ["left", "right", "shoot", "noop"]
    assert mask.tolist() == [[1, 1, 1, 1]]
    assert inputs["attention_mask"].sum(axis=1).min() > 0
    for row in range(4):
        eos = inputs["eos_map"][row, 0].nonzero()[0]
        assert len(eos) == 1
        assert inputs["input_ids"][row, eos[0]] == tokenizer.eos_token_id